fix: Exact plural forms for basic MT translators (#2454)

backend/data/src/main/kotlin/io/tolgee/service/machineTranslation/PluralTranslationUtil.kt

@@ -12,21 +12,14 @@ class PluralTranslationUtil(
  private val item: MtBatchItemParams,
  private val translateFn: (String) -> MtTranslatorResult,
 ) {
- val forms by lazy {
- context.getPluralFormsReplacingReplaceParam(baseTranslationText)
- ?: throw IllegalStateException("Plural forms are null")
- }
-
  fun translate(): MtTranslatorResult {
  return result
  }
 
  private val preparedFormSourceStrings: Sequence<Pair<String, String>> by lazy {
- return@lazy targetExamples.asSequence().map {
- val form = sourceRules?.select(it.value.toDouble())
- val formValue = forms.forms[form] ?: forms.forms[PluralRules.KEYWORD_OTHER] ?: ""
- it.key to formValue.replaceReplaceNumberPlaceholderWithExample(it.value)
- }
+ val targetLanguageTag = context.getLanguage(item.targetLanguageId).tag
+ val sourceLanguageTag = context.baseLanguage.tag
+ getPreparedSourceStrings(sourceLanguageTag, targetLanguageTag, forms)
  }
 
  private val translated by lazy {
@@ -35,6 +28,11 @@ class PluralTranslationUtil(
  }
  }
 
+ private val forms by lazy {
+ context.getPluralFormsReplacingReplaceParam(baseTranslationText)
+ ?: throw IllegalStateException("Plural forms are null")
+ }
+
  private val result: MtTranslatorResult by lazy {
  val result =
  translated.map { (form, result) ->
@@ -59,18 +57,6 @@ class PluralTranslationUtil(
  )
  }
 
- private val targetExamples by lazy {
- val targetLanguageTag = context.getLanguage(item.targetLanguageId).tag
- val targetULocale = getULocaleFromTag(targetLanguageTag)
- val targetRules = PluralRules.forLocale(targetULocale)
- getPluralFormExamples(targetRules)
- }
-
- private val sourceRules by lazy {
- val sourceLanguageTag = context.baseLanguage.tag
- getRulesByTag(sourceLanguageTag)
- }
-
  private fun String.replaceNumberTags(): String {
  return this.replace(TOLGEE_TAG_REGEX, "#")
  }
@@ -126,5 +112,43 @@ class PluralTranslationUtil(
  val sourceULocale = getULocaleFromTag(languageTag)
  return PluralRules.forLocale(sourceULocale)
  }
+
+ fun getPreparedSourceStrings(
+ sourceLanguageTag: String,
+ targetLanguageTag: String,
+ forms: PluralForms,
+ ): Sequence<Pair<String, String>> {
+ val sourceRules = getRulesByTag(sourceLanguageTag)
+ val keywordCases =
+ getTargetExamples(targetLanguageTag).asSequence().map {
+ val form = sourceRules?.select(it.value.toDouble())
+ val formValue = forms.forms[form] ?: forms.forms[PluralRules.KEYWORD_OTHER] ?: ""
+ it.key to formValue.replaceReplaceNumberPlaceholderWithExample(it.value)
+ }
+
+ val exactCases =
+ forms.forms.asSequence().filter {
+ it.key.startsWith("=")
+ }.mapNotNull {
+ val number = it.key.substring(1).toDoubleOrNull() ?: return@mapNotNull null
+ it.key to it.value.replaceReplaceNumberPlaceholderWithExample(number)
+ }
+
+ return keywordCases + exactCases
+ }
+
+ private fun String.toDoubleOrNull(): Number? {
+ return try {
+ this.toBigDecimalOrNull()
+ } catch (e: NumberFormatException) {
+ null
+ }
+ }
+
+ private fun getTargetExamples(targetLanguageTag: String): Map<String, Number> {
+ val targetULocale = getULocaleFromTag(targetLanguageTag)
+ val targetRules = PluralRules.forLocale(targetULocale)
+ return getPluralFormExamples(targetRules)
+ }
  }
 }

backend/data/src/test/kotlin/io/tolgee/unit/util/PluralTranslationUtilTest.kt

@@ -0,0 +1,31 @@
+package io.tolgee.unit.util
+
+import io.tolgee.formats.getPluralFormsReplacingReplaceParam
+import io.tolgee.service.machineTranslation.PluralTranslationUtil
+import io.tolgee.testing.assert
+import org.junit.jupiter.api.Test
+
+class PluralTranslationUtilTest {
+ @Test
+ fun `provides correct forms for basic MT providers`() {
+ val baseString = """{number, plural, one {# apple} =1 {one apple} =2 {Two apples} =5 {# apples} other {# apples}}"""
+ val result =
+ PluralTranslationUtil.getPreparedSourceStrings(
+ "en",
+ "cs",
+ getPluralFormsReplacingReplaceParam(baseString, PluralTranslationUtil.REPLACE_NUMBER_PLACEHOLDER)!!,
+ )
+
+ result.toMap().assert.isEqualTo(
+ mapOf(
+ "one" to "<x id=\"tolgee-number\">1</x> apple",
+ "few" to "<x id=\"tolgee-number\">2</x> apples",
+ "many" to "<x id=\"tolgee-number\">0.5</x> apples",
+ "other" to "<x id=\"tolgee-number\">10</x> apples",
+ "=1" to "one apple",
+ "=2" to "Two apples",
+ "=5" to "<x id=\"tolgee-number\">5</x> apples",
+ ),
+ )
+ }
+}

e2e/cypress/e2e/import/importApplication.cy.ts

@@ -22,7 +22,7 @@ describe('Import application', () => {
  'Applies import',
  {
  retries: {
- runMode: 4,
+ runMode: 10,
  },
  },
  () => {

e2e/cypress/e2e/import/importResultManupulation.cy.ts

@@ -188,7 +188,7 @@ describe('Import result & manipulation', () => {
  }
  );
 
- it('imports with selects namespaces', () => {
+ it('imports with selects namespaces', { retries: { runMode: 5 } }, () => {
  gcy('import_apply_import_button').click();
  assertMessage('Import successful');
  gcy('import-result-row').should('have.length', 0);

e2e/cypress/e2e/translations/plurals.cy.ts

@@ -8,7 +8,7 @@ import {
 } from '../../common/translations';
 import { waitForGlobalLoading } from '../../common/loading';
 import { createKey, deleteProject } from '../../common/apiCalls/common';
-import { confirmStandard } from '../../common/shared';
+import { confirmStandard, gcyAdvanced } from '../../common/shared';
 
 describe('Translations Base', () => {
  let project: ProjectDTO = null;
@@ -60,6 +60,27 @@ describe('Translations Base', () => {
  .should('be.visible');
  });
 
+ it('shows base and existing exact forms', () => {
+ createKey(
+ project.id,
+ 'Test key',
+ {
+ en: 'You have {testValue, plural, one {# item} =2 {Two items} other {# items}}',
+ cs: 'Máte {testValue, plural, one {# položku} =4 {# položky } few {# položky} other {# položek}}',
+ },
+ { isPlural: true }
+ );
+ visitTranslations(project.id);
+ waitForGlobalLoading();
+ getTranslationCell('Test key', 'cs').click();
+ gcyAdvanced({ value: 'translation-editor', variant: '=2' }).should(
+ 'be.visible'
+ );
+ gcyAdvanced({ value: 'translation-editor', variant: '=4' }).should(
+ 'be.visible'
+ );
+ });
+
  it('will change plural parameter name for all translations', () => {
  createKey(
  project.id,

webapp/src/views/projects/translations/TranslationEditor.tsx

@@ -18,6 +18,7 @@ export const TranslationEditor = ({ mode, tools, editorRef }: Props) => {
  handleSave,
  handleClose,
  handleInsertBase,
+ baseValue,
  } = tools;
 
  return (
@@ -30,6 +31,7 @@ export const TranslationEditor = ({ mode, tools, editorRef }: Props) => {
  autofocus={true}
  activeEditorRef={editorRef}
  mode={mode}
+ baseValue={baseValue}
  editorProps={{
  shortcuts: [
  { key: 'Escape', run: () => (handleClose(true), true) },

webapp/src/views/projects/translations/TranslationsTable/TranslationWrite.tsx

@@ -56,6 +56,7 @@ export const TranslationWrite: React.FC<Props> = ({ tools }) => {
  handleInsertBase,
  editEnabled,
  disabled,
+ baseText,
  } = tools;
  const editVal = tools.editVal!;
  const state = translation?.state || 'UNTRANSLATED';
@@ -68,7 +69,7 @@ export const TranslationWrite: React.FC<Props> = ({ tools }) => {
 
  const baseTranslation = useBaseTranslation(
  activeVariant,
- keyData.translations[baseLanguage]?.text,
+ baseText,
  keyData.keyIsPlural
  );
 

webapp/src/views/projects/translations/translationVisual/PluralEditor.tsx

@@ -17,6 +17,7 @@ type Props = {
  autofocus?: boolean;
  activeEditorRef?: RefObject<EditorView | null>;
  mode: 'placeholders' | 'syntax';
+ baseValue?: TolgeeFormat;
 };
 
 export const PluralEditor = ({
@@ -29,6 +30,7 @@ export const PluralEditor = ({
  activeEditorRef,
  editorProps,
  mode,
+ baseValue,
 }: Props) => {
  function handleChange(text: string, variant: string) {
  onChange?.({ ...value, variants: { ...value.variants, [variant]: text } });
@@ -38,13 +40,25 @@ export const PluralEditor = ({
 
  const editorMode = project.icuPlaceholders ? mode : 'plain';
 
+ function getExactForms() {
+ if (!baseValue) {
+ return [];
+ }
+ return Object.keys(baseValue.variants)
+ .filter((key) => /^=\d+(\.\d+)?$/.test(key))
+ .map((key) => parseFloat(key.substring(1)));
+ }
+
+ const exactForms = getExactForms();
+
  return (
  <TranslationPlurals
  value={value}
  locale={locale}
  showEmpty
  activeVariant={activeVariant}
  variantPaddingTop="8px"
+ exactForms={exactForms}
  render={({ content, variant, exampleValue }) => {
  const variantOrOther = variant || 'other';
  return (

webapp/src/views/projects/translations/translationVisual/TranslationPlurals.tsx

@@ -1,10 +1,9 @@
-import { useMemo } from 'react';
+import React, { useMemo } from 'react';
 import { styled } from '@mui/material';
-import React from 'react';
 import {
- TolgeeFormat,
  getPluralVariants,
  getVariantExample,
+ TolgeeFormat,
 } from '@tginternal/editor';
 
 const StyledContainer = styled('div')`
@@ -67,6 +66,7 @@ type Props = {
  showEmpty?: boolean;
  activeVariant?: string;
  variantPaddingTop?: number | string;
+ exactForms?: number[];
 };
 
 export const TranslationPlurals = ({
@@ -76,19 +76,12 @@ export const TranslationPlurals = ({
  showEmpty,
  activeVariant,
  variantPaddingTop,
+ exactForms,
 }: Props) => {
- const variants = useMemo(() => {
- const existing = new Set(Object.keys(value.variants));
- const required = getPluralVariants(locale);
- required.forEach((val) => existing.delete(val));
- const result = Array.from(existing).map((value) => {
- return [value, getVariantExample(locale, value)] as const;
- });
- required.forEach((value) => {
- result.push([value, getVariantExample(locale, value)]);
- });
- return result;
- }, [locale]);
+ const variants = useMemo(
+ () => getForms(locale, value, exactForms),
+ [locale, exactForms, value]
+ );
 
  if (value.parameter) {
  return (
@@ -137,3 +130,27 @@ export const TranslationPlurals = ({
  </StyledContainerSimple>
  );
 };
+
+function getForms(locale: string, value: TolgeeFormat, exactForms?: number[]) {
+ const forms: Set<string> = new Set();
+ getPluralVariants(locale).forEach((value) => forms.add(value));
+ Object.keys(value.variants).forEach((value) => forms.add(value));
+ (exactForms || [])
+ .map((value) => `=${value.toString()}`)
+ .forEach((value) => forms.add(value));
+
+ const formsArray = sortExactForms(forms);
+
+ return formsArray.map((value) => {
+ return [value, getVariantExample(locale, value)] as const;
+ });
+}
+
+function sortExactForms(forms: Set<string>) {
+ return [...forms].sort((a, b) => {
+ if (a.startsWith('=') && b.startsWith('=')) {
+ return Number(a.substring(1)) - Number(b.substring(1));
+ }
+ return 0;
+ });
+}