feat: create prefilter for failed keys (#2393)

tolgee · Jun 28, 2024 · 31924f8 · 31924f8
1 parent c26ef6d
commit 31924f8
Show file tree

Hide file tree

Showing 18 changed files with 385 additions and 187 deletions.
diff --git a/backend/api/src/main/kotlin/io/tolgee/configuration/PublicConfigurationDTO.kt b/backend/api/src/main/kotlin/io/tolgee/configuration/PublicConfigurationDTO.kt
@@ -33,6 +33,7 @@ class PublicConfigurationDTO(
   val postHogApiKey: String? = properties.postHog.apiKey
   val postHogHost: String? = properties.postHog.host
   val contentDeliveryConfigured: Boolean = properties.contentDelivery.publicUrlPrefix != null
+  val userSourceField: Boolean = properties.userSourceField
   val slack: SlackDTO =
     SlackDTO(
       enabled = (

diff --git a/...end/data/src/main/kotlin/io/tolgee/development/testDataBuilder/builders/ProjectBuilder.kt b/...end/data/src/main/kotlin/io/tolgee/development/testDataBuilder/builders/ProjectBuilder.kt
@@ -175,6 +175,7 @@ class ProjectBuilder(
   fun addWebhookConfig(ft: FT<WebhookConfig>) = addOperation(data.webhookConfigs, ft)
 
   fun addSlackConfig(ft: FT<SlackConfig>) = addOperation(data.slackConfigs, ft)
+
   fun addBatchJob(ft: FT<BatchJob>) = addOperation(data.batchJobs, ft)
 
   fun setImportSettings(ft: FT<ImportSettings>) {

diff --git a/...lin/io/tolgee/service/queryBuilders/translationViewBuilder/TranslationViewDataProvider.kt b/...lin/io/tolgee/service/queryBuilders/translationViewBuilder/TranslationViewDataProvider.kt
@@ -41,6 +41,7 @@ class TranslationViewDataProvider(
 
     // reset the value
     em.createNativeQuery("SET join_collapse_limit TO DEFAULT").executeUpdate()
+    deleteFailedKeysInJobTempTable()
 
     val keyIds = views.map { it.keyId }
     tagService.getTagsForKeyIds(keyIds).let { tagMap ->
@@ -54,16 +55,41 @@ class TranslationViewDataProvider(
       return
     }
 
-    em.createNativeQuery("select from create_unsuccessful_job_keys_temp(:jobId)")
-      .setParameter("jobId", filterFailedKeysOfJob)
-      .singleResult
+    em.createNativeQuery(
+      """
+        CREATE TEMP TABLE temp_unsuccessful_job_keys AS
+            WITH unsuccessful_targets AS (
+                SELECT *
+                FROM (
+                         SELECT jsonb_array_elements(bj.target) AS target
+                         FROM tolgee_batch_job bj
+                         WHERE bj.id = :batchJobId
+                     ) AS targets
+                WHERE targets.target NOT IN (
+                    SELECT jsonb_array_elements(tbje.success_targets)
+                    FROM tolgee_batch_job_chunk_execution tbje
+                    WHERE tbje.batch_job_id = :batchJobId
+                  )
+            )
+            SELECT DISTINCT (target -> 'keyId')\:\:bigint AS key_id
+            FROM unsuccessful_targets;
+      """,
+    )
+      .setParameter("batchJobId", filterFailedKeysOfJob)
+      .executeUpdate()
+  }
+
+  private fun deleteFailedKeysInJobTempTable() {
+    em.createNativeQuery("DROP TABLE IF EXISTS temp_unsuccessful_job_keys")
+      .executeUpdate()
   }
 
   fun getSelectAllKeys(
     projectId: Long,
     languages: Set<LanguageDto>,
     params: TranslationFilters = TranslationFilters(),
   ): MutableList<Long> {
+    createFailedKeysInJobTempTable(params.filterFailedKeysOfJob)
     val translationsViewQueryBuilder =
       TranslationsViewQueryBuilder(
         cb = em.criteriaBuilder,
@@ -73,7 +99,9 @@ class TranslationViewDataProvider(
         sort = Sort.by(Sort.Order.asc(KeyWithTranslationsView::keyId.name)),
         entityManager = em,
       )
-    return em.createQuery(translationsViewQueryBuilder.keyIdsQuery).resultList
+    val result = em.createQuery(translationsViewQueryBuilder.keyIdsQuery).resultList
+    deleteFailedKeysInJobTempTable()
+    return result
   }
 
   private fun getTranslationsViewQueryBuilder(

diff --git a/backend/data/src/main/resources/db/changelog/schema.xml b/backend/data/src/main/resources/db/changelog/schema.xml
@@ -3418,31 +3418,4 @@
     <changeSet author="huglx (generated)" id="1718704326054-19">
         <addForeignKeyConstraint baseColumnNames="project_id" baseTableName="slack_config" constraintName="FKsgau8dxyxblao9sa16fnka0t1" deferrable="false" initiallyDeferred="false" referencedColumnNames="id" referencedTableName="project" validate="true"/>
     </changeSet>
-    <changeSet author="jenik (generated)" id="1714995693669-0">
-        <sql>
-            CREATE OR REPLACE FUNCTION create_unsuccessful_job_keys_temp(batchJobId BIGINT)
-            RETURNS void
-            AS '
-                BEGIN
-                    -- Create a temporary table to store the results
-                    CREATE TEMP TABLE temp_unsuccessful_job_keys AS
-                    WITH unsuccessful_targets AS (
-                        SELECT *
-                        FROM (
-                                 SELECT jsonb_array_elements(bj.target) AS target
-                                 FROM tolgee_batch_job bj
-                                 WHERE bj.id = batchJobId
-                             ) AS targets
-                        WHERE targets.target NOT IN (
-                            SELECT jsonb_array_elements(tbje.success_targets)
-                            FROM tolgee_batch_job_chunk_execution tbje
-                            WHERE tbje.batch_job_id = batchJobId
-                          )
-                    )
-                    SELECT DISTINCT (target -> ''keyId'')::bigint AS key_id
-                    FROM unsuccessful_targets;
-                END;
-            ' LANGUAGE plpgsql;
-        </sql>
-    </changeSet>
 </databaseChangeLog>
diff --git a/webapp/src/component/activity/ActivityDetail/ActivityEntities.tsx b/webapp/src/component/activity/ActivityDetail/ActivityEntities.tsx
@@ -47,7 +47,7 @@ export const ActivityEntities: React.FC<Props> = ({
   return (
     <StyledFields>
       {activity.entities.map((entity, i) => {
-        if (!entity.fields.length) {
+        if (!entity?.fields.length) {
           return null;
         }
         return (

diff --git a/webapp/src/hooks/ProjectContext.tsx b/webapp/src/hooks/ProjectContext.tsx
@@ -78,8 +78,10 @@ export const [ProjectContext, useProjectActions, useProjectContext] =
 
     const changeHandler = ({ data }: BatchJobProgress) => {
       const exists = batchOperations?.find((job) => job.id === data.jobId);
+      let shouldRefetch = false;
       if (!exists) {
         if (!knownJobs.includes(data.jobId)) {
+          shouldRefetch = true;
           // only refetch jobs first time we see unknown job
           setKnownJobs((jobs) => [...jobs, data.jobId]);
           setBatchOperations((jobs) => [
@@ -92,12 +94,15 @@ export const [ProjectContext, useProjectActions, useProjectContext] =
               errorMessage: data.errorMessage,
             },
           ]);
-          batchJobsLoadable.refetch();
         }
       } else {
         setBatchOperations((jobs) =>
           jobs?.map((job) => {
             if (job.id === data.jobId) {
+              if (data.status === 'FAILED' && data.status !== job.status) {
+                // load error message
+                shouldRefetch = true;
+              }
               return {
                 ...job,
                 totalItems: data.total ?? job.totalItems,
@@ -110,6 +115,9 @@ export const [ProjectContext, useProjectActions, useProjectContext] =
           })
         );
       }
+      if (shouldRefetch) {
+        batchJobsLoadable.refetch({ fetching: true });
+      }
     };
 
     const changeHandlerRef = useRef(changeHandler);