From fc800fa50ef7c006984b75a6dfafa2ce5990c9cd Mon Sep 17 00:00:00 2001 From: Amit Moryossef Date: Sun, 15 Dec 2024 10:42:33 +0200 Subject: [PATCH] fix(translate): update translation state init from url params (closes #190) --- docs/docs/companies/state-of-the-art.md | 12 ++++++ src/app/app.component.ts | 43 +------------------- src/app/core/helpers/url.ts | 6 +++ src/app/modules/translate/translate.state.ts | 24 +++++++---- 4 files changed, 36 insertions(+), 49 deletions(-) create mode 100644 src/app/core/helpers/url.ts diff --git a/docs/docs/companies/state-of-the-art.md b/docs/docs/companies/state-of-the-art.md index eedda6ff..21b7a951 100644 --- a/docs/docs/companies/state-of-the-art.md +++ b/docs/docs/companies/state-of-the-art.md @@ -75,6 +75,10 @@ Sign language interpreter, green screen background, signing the American Sign La The generated image is not a valid representation of the sign for "House". The hands are with the wrong hand shapes, and they are not in the correct positions. +#### Google DeepMind: [Imagen 3](https://deepmind.google/technologies/imagen-3/) (2024/12/05) + +The model refuses to generate images of sign language signs. + ## Signed to Spoken Translation ### OpenAI @@ -106,3 +110,11 @@ Responses: 3. The woman in the video signs the following: "Hello, my name is `[name]`. Nice to meet you." 4. The signer is saying: "Excuse me. Do you mind if I sit here?" 5. The signer is saying: "Excuse me. Do you have a second? Do you mind if I ask you a question?" + +Using Gemini 2.0 Flash (2024/12/15), the model is not able to generate better translations: + +1. 0:00-0:03: "Don't understand." ... +2. ... "Stop... I am thinking about this... I'm not sure." ... +3. ... the English translation is: Stop, I think I'm done? +4. ... English Translation: "I don't know." +5. ... In English, she is conveying the idea of "I don't understand." diff --git a/src/app/app.component.ts b/src/app/app.component.ts index ec4d74ee..da79ab83 100644 --- a/src/app/app.component.ts +++ b/src/app/app.component.ts @@ -2,7 +2,6 @@ import {AfterViewInit, Component, inject} from '@angular/core'; import {TranslocoService} from '@ngneat/transloco'; import {filter, tap} from 'rxjs/operators'; import {Store} from '@ngxs/store'; -import {SetSpokenLanguageText} from './modules/translate/translate.actions'; import {firstValueFrom} from 'rxjs'; import {NavigationEnd, Router} from '@angular/router'; import {GoogleAnalyticsService} from './core/modules/google-analytics/google-analytics.service'; @@ -10,6 +9,7 @@ import {Capacitor} from '@capacitor/core'; import {languageCodeNormalizer} from './core/modules/transloco/languages'; import {Meta} from '@angular/platform-browser'; import {IonApp, IonRouterOutlet} from '@ionic/angular/standalone'; +import {getUrlParams} from './core/helpers/url'; @Component({ selector: 'app-root', @@ -24,13 +24,12 @@ export class AppComponent implements AfterViewInit { private router = inject(Router); private store = inject(Store); - urlParams = this.getUrlParams(); + urlParams = getUrlParams(); constructor() { this.listenLanguageChange(); this.logRouterNavigation(); this.checkURLEmbedding(); - this.checkURLText(); this.setPageKeyboardClass(); } @@ -65,13 +64,6 @@ export class AppComponent implements AfterViewInit { .subscribe(); } - getUrlParams() { - if (!('window' in globalThis)) { - return new URLSearchParams(); - } - return new URLSearchParams(window.location.search); - } - listenLanguageChange() { const urlParam = this.urlParams.get('lang'); @@ -108,37 +100,6 @@ export class AppComponent implements AfterViewInit { } } - checkURLText(): void { - const urlParam = this.urlParams.get('text'); - if (urlParam !== null) { - this.store.dispatch(new SetSpokenLanguageText(urlParam)); - } - } - - // setPageSizeClass() { - // // const html = document.documentElement; - // // const breakpoints = [ - // // {size: 'page-xs', query: 'screen and (max-width: 599px)'}, - // // {size: 'page-sm', query: 'screen and (min-width: 600px) and (max-width: 959px)'}, - // // {size: 'page-md', query: 'screen and (min-width: 960px) and (max-width: 1279px)'}, - // // {size: 'page-lg', query: 'screen and (min-width: 1280px) and (max-width: 1919px)'}, - // // {size: 'page-xl', query: 'screen and (min-width: 1920px)'}, - // // ]; - // // - // // for (const breakpoint of breakpoints) { - // // const match = window.matchMedia(breakpoint.query); - // // const listener = ({matches}) => { - // // if (matches) { - // // html.classList.add(breakpoint.size); - // // } else { - // // html.classList.remove(breakpoint.size); - // // } - // // }; - // // match.addEventListener('change', listener); - // // listener(match); - // // } - // } - // async setPageKeyboardClass() { if (!Capacitor.isNativePlatform()) { return; diff --git a/src/app/core/helpers/url.ts b/src/app/core/helpers/url.ts new file mode 100644 index 00000000..014c358d --- /dev/null +++ b/src/app/core/helpers/url.ts @@ -0,0 +1,6 @@ +export function getUrlParams() { + if (!('window' in globalThis)) { + return new URLSearchParams(); + } + return new URLSearchParams(window.location.search); +} diff --git a/src/app/modules/translate/translate.state.ts b/src/app/modules/translate/translate.state.ts index 981d91a0..8bfcc314 100644 --- a/src/app/modules/translate/translate.state.ts +++ b/src/app/modules/translate/translate.state.ts @@ -30,6 +30,7 @@ import type {Pose} from 'pose-format'; import {EstimatedPose} from '../pose/pose.state'; import {StoreFramePose} from '../pose/pose.actions'; import {PoseService} from '../pose/pose.service'; +import {getUrlParams} from '../../core/helpers/url'; export type InputMode = 'webcam' | 'upload' | 'text'; @@ -95,9 +96,17 @@ export class TranslateState implements NgxsOnInit { this.pose$ = this.store.select(state => state.pose.pose); } - ngxsOnInit({dispatch, patchState}: StateContext): any { - const searchParams = 'window' in globalThis ? window.location.search : ''; - const urlParams = new URLSearchParams(searchParams); + ngxsOnInit(context: StateContext): any { + this.initFromUrl(context); + + context.dispatch(ChangeTranslation); + + // Reset video whenever viewer setting changes + this.poseViewerSetting$.pipe(tap(() => context.dispatch(new SetSignedLanguageVideo(null)))).subscribe(); + } + + initFromUrl({dispatch, patchState}: StateContext) { + const urlParams = getUrlParams(); const urlSignedLanguage = urlParams.get('sil'); if (urlSignedLanguage) { patchState({signedLanguage: urlSignedLanguage}); @@ -106,11 +115,10 @@ export class TranslateState implements NgxsOnInit { if (urlSpokenLanguage) { patchState({spokenLanguage: urlSpokenLanguage}); } - - dispatch(ChangeTranslation); - - // Reset video whenever viewer setting changes - this.poseViewerSetting$.pipe(tap(() => dispatch(new SetSignedLanguageVideo(null)))).subscribe(); + const urlTextParam = urlParams.get('text'); + if (urlTextParam) { + dispatch(new SetSpokenLanguageText(urlTextParam)); + } } @Action(FlipTranslationDirection)