diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index fc2af2b7..66908075 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -50,6 +50,20 @@ export default withMermaid( }, ], }, + { + text: 'Technology', + collapsed: false, + items: [ + { + text: 'Our Approach', + link: '/docs/technology/introduction', + }, + { + text: 'Reviews & Awards', + link: '/docs/technology/awards', + }, + ], + }, { text: 'Fact Sheets', collapsed: false, diff --git a/docs/docs/technology/assets/dictionary-pipeline.png b/docs/docs/technology/assets/dictionary-pipeline.png new file mode 100644 index 00000000..c34306e7 Binary files /dev/null and b/docs/docs/technology/assets/dictionary-pipeline.png differ diff --git a/docs/docs/technology/assets/sign-tube-example.png b/docs/docs/technology/assets/sign-tube-example.png new file mode 100644 index 00000000..fa421abf Binary files /dev/null and b/docs/docs/technology/assets/sign-tube-example.png differ diff --git a/docs/docs/technology/awards.md b/docs/docs/technology/awards.md new file mode 100644 index 00000000..a9b07ea8 --- /dev/null +++ b/docs/docs/technology/awards.md @@ -0,0 +1,29 @@ +# Awards and Recognition + +In this page, we list the awards and recognition that the technology behind sign.mt has received in the last few years, alongside public independent reviews. + +## 2024 + +- We received an `EMNLP Outstanding Demo Paper Award` for our demonstration of "sign.mt: Real-Time Multilingual Sign Language Translation Application"[^emnlp2024-award]. + +[^emnlp2024-award]: sign.mt. 2024. [Twitter Post](https://x.com/signmt_/status/1857181686045540787). + +- Dr. Amit Moryossef (among others) received the `SwissNLP Award` for his "outstanding contribution to Sign Language Translation in Switzerland"[^swissnlp2024-award]. + +[^swissnlp2024-award]: SwissNLP. 2024. [SwissNLP Award](https://swissnlp.org/home/activities/swissnlp-award/). + +- sign.mt was independently reviewed for Kazakh-Russian Sign Language translation, reporting accuracy of 37% compared to 98% for human translation[^kazakh-russian-review]. + +[^kazakh-russian-review]: Imashev et al. 2024. [Comparative Analysis of Sign Language Interpreting Agents Perception: A Study of the Deaf](hhttps://aclanthology.org/2024.lrec-main.319/). + +## 2023 + +- Dr. Amit Moryossef (among others) received the `ACL Outstanding Paper Award` for "Considerations for Meaningful Sign Language Machine Translation"[^acl2023-award]. + +[^acl2023-award]: Müller et al. 2023. [Considerations for Meaningful Sign Language Machine Translation](https://aclanthology.org/2023.acl-short.60/). + +## 2021 + +- Dr. Amit Moryossef (among others) received the `ACL Best Theme Paper Award` for "Including Signed Languages in Natural Language Processing"[^acl2021-award]. + +[^acl2021-award]: Yin et al. 2021. [Including Signed Languages in Natural Language Processing](https://aclanthology.org/2021.acl-long.570/). diff --git a/docs/docs/technology/introduction.md b/docs/docs/technology/introduction.md new file mode 100644 index 00000000..6aeda82b --- /dev/null +++ b/docs/docs/technology/introduction.md @@ -0,0 +1,126 @@ +# Our Approach + +Following the research of Dr. Amit Moryosef's published in his PhD thesis[^amit-thesis], we aim to develop a sign language translation system that separates the computer vision tasks from the language translation tasks. + +[^amit-thesis]: Amit Moryosef. 2024. [Real-Time Multilingual Sign Language Processing](https://arxiv.org/abs/2412.01991). + +## Spoken to Signed Language Translation + +Following, is a flowchart of the current translation pipeline from spoken to signed language. +Each node represents a different module or function in the pipeline, with a link to the relevant code repository. + +- Green edges represent high quality modules. +- orange edges represent existing, low quality modules. +- red edges represent modules that need to be implemented. + +```mermaid +flowchart TD + A0[Spoken Language Audio] --> A1(Spoken Language Text) + A1[Spoken Language Text] --> B[Language Identification] + A1 --> C(Normalized Text) + B --> C + C & B --> Q(Sentence Splitter) + Q & B --> D(SignWriting) + C -.-> M(Glosses) + M -.-> E + D --> E(Pose Sequence) + D -.-> I(Illustration) + N --> H(3D Avatar) + N --> G(Skeleton Viewer) + N --> F(Human GAN) + H & G & F --> J(Video) + J --> K(Share Translation) + D -.-> L(Description) + O --> N(Fluent Pose Sequence) + E --> O(Pose Appearance Transfer) + +linkStyle default stroke:green; +linkStyle 3,5,7 stroke:lightgreen; +linkStyle 10,11,12,15 stroke:red; +linkStyle 6,8,9,14,19,20 stroke:orange; +``` + +This pipeline in fact represents two types of approaches to translation: + +1. **Dictionary Based Translation** +2. **SignWriting based Machine Translation** + +### Dictionary Based Translation[^dictionary-baseline] + +[^dictionary-baseline]: Moryossef et al. 2023. [An Open-Source Gloss-Based Baseline for Spoken to Signed Language Translation](https://aclanthology.org/2023.at4ssl-1.3/). + +The dictionary-based translation approach aims to simplify the translation but sacrifices the fluency and natural expressiveness needed for accurate sign language communication. It can be characterized by the following general steps: + +```mermaid +flowchart LR + a[Spoken Language Text] --> b[Glosses] --> c[Pose Sequence] --> d[Video] +``` + +![Visualization of one example through the dictionary-based translation pipeline](./assets//dictionary-pipeline.png) + +#### **Main Translation Steps** + +1. **text-to-gloss translation**: The input spoken language text is transformed into glosses using techniques such as lemmatization, word reordering, and dropping unnecessary words (like articles). + +2. **gloss-to-pose conversion**: ‘Glosses’ are mapped to corresponding skeletal poses extracted from a sign language dictionary. + +3. **pose-to-video rendering**: The pose sequences are stitched together and rendered back into a synthesized human video. + +#### **Data Requirements** + +A dictionary of sign language videos for isolated letters, words, and phrases. This dictionary cannot represent the full expressiveness of sign language. + +#### **Potential Quality** + +Even if the dictionary is complete, this approach cannot fully capture the complexity of sign language. While it attempts to translate from text to a gloss-based structure and map that to movements (poses), it fails to account for the full grammatical and syntactic richness of sign language. Glosses are an incomplete representation of actual signs, and grammar in sign language differs substantially from spoken languages. + +### SignWriting based Machine Translation + +The machine translation approach aims to achieve similar translation quality to spoken language translation systems such as Google Translate, with potentially high fluency and natural expressiveness. This approach further allows for bi-directional translation, between signed and spoken languages. + +```mermaid +flowchart LR + a[Spoken Language Text] --> b[SignWriting] --> c[Pose Sequence] --> d[Video] +``` + +![Visualization of one example through the SignWriting-based translation pipeline](./assets/sign-tube-example.png) + +#### **Main Translation Steps:** + +1. **text-to-SignWriting Translation**: Spoken language text is translated into SignWriting sequences using machine translation (it sees a lot of data, and extrapolates from it). + +2. **SignWriting-to-pose conversion**: The written signs are converted into a fluent pose sequence, illustrating how the signs would be physically performed by a signer. + +3. **pose-to-video rendering**: This pose sequence is then rendered into a human video. + +#### **Data Requirements:** + +By combining a relatively small dataset of transcribed single signs (~100k) with a relatively small dataset of segmented continuous signs, and leveraging large video/text sign language datasets, we can automatically transcribe the latter. This process will generate large synthesized datasets for both **text-to-SignWriting** and **SignWriting-to-pose** conversions. + +#### **Potential Quality:** + +The system aims to accurately represent sign language grammar and structure, allowing for a good translation of both lexical and non-lexical signs, expressions, and classifiers. Potentially, the system can be as good as a deaf human translator, given quality data. + +## Signed to Spoken Language Translation + +Following, is a flowchart of the current translation pipeline from signed to spoken language. + +```mermaid +flowchart TD + A0[Upload Sign Language Video] --> A3[Video] + A1[Camera Sign Language Video] --> A3 + A3 --> B(Pose Estimation) + B --> C(Segmentation) + C & B --> D(SignWriting Transcription) + A2[Language Selector] --> E(Spoken Language Text) + D --> E + E --> F(Spoken Language Audio) + E --> G(Share Translation) + C -.-> H(Sign Image) + + +linkStyle 1,2 stroke:orange; +linkStyle 4,5,6 stroke:lightgreen; +linkStyle 3,5,7,10 stroke:red; +linkStyle 0,8,9 stroke:green; +``` diff --git a/src/app/modules/settings/settings/settings.component.html b/src/app/modules/settings/settings/settings.component.html index 4eadac22..ad2c8531 100644 --- a/src/app/modules/settings/settings/settings.component.html +++ b/src/app/modules/settings/settings/settings.component.html @@ -1,4 +1,4 @@ - + @for (s of availableSettings; track s) { diff --git a/src/app/pages/landing/about/about-numbers/about-numbers.component.html b/src/app/pages/landing/about/about-numbers/about-numbers.component.html new file mode 100644 index 00000000..77f5a9f4 --- /dev/null +++ b/src/app/pages/landing/about/about-numbers/about-numbers.component.html @@ -0,0 +1,22 @@ +
+

Our Vision: To Improve the Lives of 70 Million Deaf People Worldwide

+
+ @for (fact of facts; track fact) { + + + + {{ fact.title }} + @if (fact.source) { + + source + + } + + + +

{{ fact.description }}

+ + + } +
+
diff --git a/src/app/pages/landing/about/about-numbers/about-numbers.component.scss b/src/app/pages/landing/about/about-numbers/about-numbers.component.scss new file mode 100644 index 00000000..e0eb7f0c --- /dev/null +++ b/src/app/pages/landing/about/about-numbers/about-numbers.component.scss @@ -0,0 +1,28 @@ +@import '../../../../../theme/variables'; + +h2 { + text-align: center; +} + +.cards-container { + display: grid; + grid-template-columns: repeat(3, 1fr); + + @media #{$breakpoint-lt-md} { + grid-template-columns: repeat(2, 1fr); + } + + @media #{$breakpoint-lt-sm} { + grid-template-columns: 1fr; + } +} + +ion-card-title { + display: flex; + align-items: center; + + a { + flex-grow: 1; + text-align: end; + } +} diff --git a/src/app/pages/landing/about/about-numbers/about-numbers.component.spec.ts b/src/app/pages/landing/about/about-numbers/about-numbers.component.spec.ts new file mode 100644 index 00000000..76f9f812 --- /dev/null +++ b/src/app/pages/landing/about/about-numbers/about-numbers.component.spec.ts @@ -0,0 +1,31 @@ +import {ComponentFixture, TestBed} from '@angular/core/testing'; + +import {AboutNumbersComponent} from './about-numbers.component'; +import {provideIonicAngular} from '@ionic/angular/standalone'; +import {axe, toHaveNoViolations} from 'jasmine-axe'; + +describe('AboutNumbersComponent', () => { + let component: AboutNumbersComponent; + let fixture: ComponentFixture; + + beforeEach(async () => { + await TestBed.configureTestingModule({ + imports: [AboutNumbersComponent], + providers: [provideIonicAngular()], + }).compileComponents(); + + fixture = TestBed.createComponent(AboutNumbersComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); + + it('should pass accessibility test', async () => { + jasmine.addMatchers(toHaveNoViolations); + const a11y = await axe(fixture.nativeElement); + expect(a11y).toHaveNoViolations(); + }); +}); diff --git a/src/app/pages/landing/about/about-numbers/about-numbers.component.ts b/src/app/pages/landing/about/about-numbers/about-numbers.component.ts new file mode 100644 index 00000000..e5f4fb2e --- /dev/null +++ b/src/app/pages/landing/about/about-numbers/about-numbers.component.ts @@ -0,0 +1,53 @@ +import {Component} from '@angular/core'; +import {IonBadge, IonCard, IonCardContent, IonCardHeader, IonCardTitle} from '@ionic/angular/standalone'; + +@Component({ + selector: 'app-about-numbers', + standalone: true, + imports: [IonCard, IonCardHeader, IonCardTitle, IonCardContent, IonBadge], + templateUrl: './about-numbers.component.html', + styleUrl: './about-numbers.component.scss', +}) +export class AboutNumbersComponent { + facts = [ + { + title: '70,000,000', + description: + 'Globally there are 70 million deaf people. By 2050, over 700 million people could have disabling hearing loss.', + source: 'https://sign.mt/docs/docs/facts/population.html#deafness', + color: 'primary', + }, + { + title: 'Literacy Challenge', + description: + 'Spoken language literacy rates among deaf individuals are uncertain, with estimates ranging from 20% to 50%.', + source: 'https://sign.mt/docs/docs/facts/literacy.html', + color: 'warning', + }, + { + title: '$33.7B', + description: + 'The U.S. sign language translation market is worth $11B, with a global estimate of $33.7B across 30 developed nations.', + source: 'https://sign.mt/docs/docs/facts/market.html', + color: 'success', + }, + { + title: '3x 1million+', + description: 'Our technology is being reviewed by three enterprises, each with over 1 million retail customers.', + }, + { + title: '+200 Languages', + description: + 'Our translation solution currently partially supports 43 of the 200+ major sign languages to some extent, with exciting plans to expand further.', + source: 'https://sign.mt/docs/docs/facts/population.html#deafness', + color: 'tertiary', + }, + { + title: 'Reviews and Awards', + description: + 'Independently reviewed with multiple awards, praised for innovation and effectiveness in Natural Language Processing.', + source: 'https://sign.mt/docs/docs/technology/awards.html', + color: 'danger', + }, + ]; +} diff --git a/src/app/pages/landing/about/about.component.html b/src/app/pages/landing/about/about.component.html index bbaeacdc..95b34a22 100644 --- a/src/app/pages/landing/about/about.component.html +++ b/src/app/pages/landing/about/about.component.html @@ -2,6 +2,7 @@ + diff --git a/src/app/pages/landing/about/about.component.spec.ts b/src/app/pages/landing/about/about.component.spec.ts index fbae18e5..9337a72e 100644 --- a/src/app/pages/landing/about/about.component.spec.ts +++ b/src/app/pages/landing/about/about.component.spec.ts @@ -22,6 +22,7 @@ import {NoopAnimationsModule} from '@angular/platform-browser/animations'; import {AboutPricingComponent} from './about-pricing/about-pricing.component'; import {ReactiveFormsModule} from '@angular/forms'; import {LazyMapComponent} from './lazy-map/lazy-map.component'; +import {AboutNumbersComponent} from './about-numbers/about-numbers.component'; describe('AboutComponent', () => { let component: AboutComponent; @@ -45,6 +46,7 @@ describe('AboutComponent', () => { LazyMapComponent, ], imports: [ + AboutNumbersComponent, AppTranslocoTestingModule, MatTooltipModule, MatTabsModule, diff --git a/src/app/pages/landing/landing.module.ts b/src/app/pages/landing/landing.module.ts index db111522..3f38e903 100644 --- a/src/app/pages/landing/landing.module.ts +++ b/src/app/pages/landing/landing.module.ts @@ -32,6 +32,7 @@ import {ReactiveFormsModule} from '@angular/forms'; import {register} from 'swiper/element/bundle'; import {SignedToSpokenModule} from '../translate/signed-to-spoken/signed-to-spoken.module'; import {LogoComponent} from '../../components/logo/logo.component'; +import {AboutNumbersComponent} from './about/about-numbers/about-numbers.component'; register(); @@ -71,6 +72,7 @@ register(); MermaidChartComponent, ReactiveFormsModule, SignedToSpokenModule, + AboutNumbersComponent, ], bootstrap: [LandingComponent], exports: [I18NLanguageSelectorComponent],