Merge pull request #645 from alpheios-project/i644-language-detect-ad…

…apter

Add adapter for language detection

packages/client-adapters/package.json

@@ -8,7 +8,7 @@
  "test": "jest --coverage && shx cat ./coverage/lcov.info | coveralls",
  "test-no-coverage": "jest",
  "test-c": "jest tests/client-adapters.test.js",
- "test-u": "jest tests/adapters/dtsapi/adapter.test.js",
+ "test-u": "jest tests/adapters/detectlang/adapter.test.js --coverage",
  "test-l": "jest tests/adapters/logeion/adapter.test.js",
  "test-a": "jest tests/adapters/arethusa/adapter.test.js",
  "build": "npm run build-lib && npm run build-node",

packages/client-adapters/src/adapters/adapters-config.json

@@ -99,5 +99,15 @@
  "getDocument": ["baseUrl", "id"]
  }
  }
+ },
+
+ "detectlangGroup": {
+ "detectlang": {
+ "adapter": "detectLangMethod",
+ "methods": [ "getDetectedLangsList" ],
+ "params": {
+ "getDetectedLangsList": ["text"]
+ }
+ }
  }
 }

packages/client-adapters/src/adapters/detectlang/adapter.js

@@ -0,0 +1,83 @@
+import BaseAdapter from '@clAdapters/adapters/base-adapter'
+import DefaultConfig from '@clAdapters/adapters/detectlang/config.json'
+import LangsList from '@clAdapters/adapters/detectlang/langs-list.json'
+
+export default class DetectLangAdapter extends BaseAdapter {
+ /**
+ *
+ * @param {Object} config - properties for the adapter
+ */
+ constructor (config = {}) {
+ super()
+ this.config = this.uploadConfig(config, DefaultConfig)
+ this.sourceData = config.sourceData
+ }
+
+ /**
+ *
+ * @param {String} text - text for analysis
+ * @returns {String} - langCode ISO 639-3 - a detected language
+ */
+ async getDetectedLangsList (text) {
+ try {
+ const requestParams = {
+ method: 'POST',
+ headers: { Authorization: `Bearer ${this.config.api}` }
+ }
+
+ const url = this.getUrl(text)
+ if (!url) {
+ this.addError(this.l10n.getMsg('DETECT_LANG_URL_ERROR'))
+ return
+ }
+
+ let langsData
+ if (this.sourceData) {
+ langsData = this.sourceData
+ } else {
+ langsData = await this.fetch(url, { requestParams })
+ }
+ return this.chooseOneLanguage(langsData)
+ } catch (error) {
+ this.addError(this.l10n.getMsg('DETECT_LANG_FETCH_ERROR', { message: error.message }))
+ }
+ }
+
+ /**
+ *
+ * @param {String} text - text for analysis
+ * @returns {String} - constructed URL
+ */
+ getUrl (text) {
+ if (text) {
+ return `${this.config.baseurl}?q=${encodeURIComponent(text)}`
+ }
+ return null
+ }
+
+ /**
+ * The remote service returns the following format
+ * { data: {
+ detections: [
+ { language: 'en', isReliable: true, confidence: 3.36 },
+ { language: 'pt', isReliable: false, confidence: 3.36 },
+ { language: 'eu', isReliable: false, confidence: 3.36 }
+ ]
+ }}
+ * We need return only one the most reliable languageCode in ISO 639-3 format
+ * @param {Object} langsData
+ * @returns {String|null} lang code in ISO 639-3
+ */
+ chooseOneLanguage (langsData) {
+ if (langsData && langsData.data && langsData.data.detections && langsData.data.detections.length > 0) {
+ const reliableLangs = langsData.data.detections
+ .filter(langItem => langItem.isReliable)
+ if (reliableLangs && (reliableLangs.length > 0)) {
+ const lang = reliableLangs.sort((a, b) => a.confidence - b.confidence)
+ .reverse()[0].language
+ return LangsList[lang] ? LangsList[lang].langCode : lang
+ }
+ }
+ return null
+ }
+}

packages/client-adapters/src/adapters/detectlang/config.json

@@ -0,0 +1,4 @@
+{
+ "baseurl": "https://ws.detectlanguage.com/0.2/detect",
+ "api": "30e63bcc426af7b7c650aab568ed9ad7"
+}

packages/client-adapters/src/adapters/detectlang/langs-list.json

@@ -0,0 +1,166 @@
+{
+ "aa": { "label": "Afar", "langCode": "aar" },
+ "ab": { "label": "Abkhazian", "langCode": "abk" },
+ "af": { "label": "Afrikaans", "langCode": "afr" },
+ "ak": { "label": "Akan", "langCode": "aka" },
+ "am": { "label": "Amharic", "langCode": "amh" },
+ "ar": { "label": "Arabic", "langCode": "ara" },
+ "as": { "label": "Assamese", "langCode": "asm" },
+ "ay": { "label": "Aymara", "langCode": "aym" },
+ "az": { "label": "Azerbaijani", "langCode": "aze" },
+ "ba": { "label": "Bashkir", "langCode": "bak" },
+ "be": { "label": "Belarusian", "langCode": "bel" },
+ "bg": { "label": "Bulgarian", "langCode": "bul" },
+ "bh": { "label": "Bihari", "langCode": "bih" },
+ "bi": { "label": "Bislama", "langCode": "bis" },
+ "bn": { "label": "Bengali", "langCode": "ben" },
+ "bo": { "label": "Tibetan", "langCode": "bod" },
+ "br": { "label": "Breton", "langCode": "bre" },
+ "bs": { "label": "Bosnian", "langCode": "bos" },
+ "bug": { "label": "Buginese", "langCode": "bug" },
+ "ca": { "label": "Catalan", "langCode": "cat" },
+ "ceb": { "label": "Cebuano", "langCode": "ceb" },
+ "chr": { "label": "Cherokee", "langCode": "chr" },
+ "co": { "label": "Corsican", "langCode": "cos" },
+ "crs": { "label": "Seselwa", "langCode": "crs" },
+ "cs": { "label": "Czech", "langCode": "ces" },
+ "cy": { "label": "Welsh", "langCode": "cym" },
+ "da": { "label": "Danish", "langCode": "dan" },
+ "de": { "label": "German", "langCode": "deu" },
+ "dv": { "label": "Dhivehi", "langCode": "div" },
+ "dz": { "label": "Dzongkha", "langCode": "dzo" },
+ "egy": { "label": "Egyptian", "langCode": "egy" },
+ "el": { "label": "Greek", "langCode": "grc" },
+ "en": { "label": "English", "langCode": "eng" },
+ "eo": { "label": "Esperanto", "langCode": "epo" },
+ "es": { "label": "Spanish", "langCode": "spa" },
+ "et": { "label": "Estonian", "langCode": "est" },
+ "eu": { "label": "Basque", "langCode": "eus" },
+ "fa": { "label": "Persian", "langCode": "per" },
+ "fi": { "label": "Finnish", "langCode": "fin" },
+ "fj": { "label": "Fijian", "langCode": "fij" },
+ "fo": { "label": "Faroese", "langCode": "fao" },
+ "fr": { "label": "French", "langCode": "fra" },
+ "fy": { "label": "Frisian", "langCode": "frr" },
+ "ga": { "label": "Irish", "langCode": "gle" },
+ "gd": { "label": "Scots Gaelic", "langCode": "gla" },
+ "gl": { "label": "Galician", "langCode": "glg" },
+ "gn": { "label": "Guarani", "langCode": "grn" },
+ "got": { "label": "Gothic", "langCode": "got" },
+ "gu": { "label": "Gujarati", "langCode": "guj" },
+ "gv": { "label": "Manx", "langCode": "glv" },
+ "ha": { "label": "Hausa", "langCode": "hau" },
+ "haw": { "label": "Hawaiian", "langCode": "haw" },
+ "hi": { "label": "Hindi", "langCode": "hin" },
+ "hmn": { "label": "Hmong", "langCode": "hmn" },
+ "hr": { "label": "Croatian", "langCode": "hrv" },
+ "ht": { "label": "Haitian Creole", "langCode": "hat" },
+ "hu": { "label": "Hungarian", "langCode": "hun" },
+ "hy": { "label": "Armenian", "langCode": "hye" },
+ "ia": { "label": "Interlingua", "langCode": "ina" },
+ "id": { "label": "Indonesian", "langCode": "ind" },
+ "ie": { "label": "Interlingue", "langCode": "ile" },
+ "ig": { "label": "Igbo", "langCode": "ibo" },
+ "ik": { "label": "Inupiaq", "langCode": "ipk" },
+ "is": { "label": "Icelandic", "langCode": "isl" },
+ "it": { "label": "Italian", "langCode": "ita" },
+ "iu": { "label": "Inuktitut", "langCode": "iku" },
+ "iw": { "label": "Hebrew", "langCode": "heb" },
+ "ja": { "label": "Japanese", "langCode": "jpn" },
+ "jw": { "label": "Javanese", "langCode": "jav" },
+ "ka": { "label": "Georgian", "langCode": "kat" },
+ "kha": { "label": "Khasi", "langCode": "kha" },
+ "kk": { "label": "Kazakh", "langCode": "kaz" },
+ "kl": { "label": "Greenlandic", "langCode": "kal" },
+ "km": { "label": "Khmer", "langCode": "khm" },
+ "kn": { "label": "Kannada", "langCode": "kan" },
+ "ko": { "label": "Korean", "langCode": "kor" },
+ "ks": { "label": "Kashmiri", "langCode": "kas" },
+ "ku": { "label": "Kurdish", "langCode": "kur" },
+ "ky": { "label": "Kyrgyz", "langCode": "kir" },
+ "la": { "label": "Latin", "langCode": "lat" },
+ "lb": { "label": "Luxembourgish", "langCode": "ltz" },
+ "lg": { "label": "Ganda", "langCode": "lug" },
+ "li": { "label": "Limbu", "langCode": "lim" },
+ "ln": { "label": "Lingala", "langCode": "lin" },
+ "lo": { "label": "Laothian", "langCode": "lao" },
+ "lt": { "label": "Lithuanian", "langCode": "lit" },
+ "lv": { "label": "Latvian", "langCode": "lav" },
+ "mfe": { "label": "Mauritian Creole", "langCode": "mfe" },
+ "mg": { "label": "Malagasy", "langCode": "mlg" },
+ "mi": { "label": "Maori", "langCode": "mao" },
+ "mk": { "label": "Macedonian", "langCode": "mac" },
+ "ml": { "label": "Malayalam", "langCode": "mal" },
+ "mn": { "label": "Mongolian", "langCode": "mon" },
+ "mr": { "label": "Marathi", "langCode": "mar" },
+ "ms": { "label": "Malay", "langCode": "msa" },
+ "mt": { "label": "Maltese", "langCode": "mlt" },
+ "my": { "label": "Burmese", "langCode": "mya" },
+ "na": { "label": "Nauru", "langCode": "nau" },
+ "ne": { "label": "Nepali", "langCode": "nep" },
+ "nl": { "label": "Dutch", "langCode": "nld" },
+ "no": { "label": "Norwegian", "langCode": "nor" },
+ "nr": { "label": "Ndebele", "langCode": "nbl" },
+ "nso": { "label": "Pedi", "langCode": "nso" },
+ "ny": { "label": "Nyanja", "langCode": "nya" },
+ "oc": { "label": "Occitan", "langCode": "oci" },
+ "om": { "label": "Oromo", "langCode": "orm" },
+ "or": { "label": "Oriya", "langCode": "ori" },
+ "pa": { "label": "Punjabi", "langCode": "pan" },
+ "pl": { "label": "Polish", "langCode": "pol" },
+ "ps": { "label": "Pashto", "langCode": "pus" },
+ "pt": { "label": "Portuguese", "langCode": "por" },
+ "qu": { "label": "Quechua", "langCode": "que" },
+ "rm": { "label": "Rhaeto Romance", "langCode": "roh" },
+ "rn": { "label": "Rundi", "langCode": "run" },
+ "ro": { "label": "Romanian", "langCode": "ron" },
+ "ru": { "label": "Russian", "langCode": "rus" },
+ "rw": { "label": "Kinyarwanda", "langCode": "kin" },
+ "sa": { "label": "Sanskrit", "langCode": "san" },
+ "sco": { "label": "Scots", "langCode": "sco" },
+ "sd": { "label": "Sindhi", "langCode": "snd" },
+ "sg": { "label": "Sango", "langCode": "sag" },
+ "si": { "label": "Sinhalese", "langCode": "sin" },
+ "sk": { "label": "Slovak", "langCode": "slk" },
+ "sl": { "label": "Slovenian", "langCode": "slv" },
+ "sm": { "label": "Samoan", "langCode": "smo" },
+ "sn": { "label": "Shona", "langCode": "sna" },
+ "so": { "label": "Somali", "langCode": "som" },
+ "sq": { "label": "Albanian", "langCode": "sqi" },
+ "sr": { "label": "Serbian", "langCode": "srp" },
+ "ss": { "label": "Siswant", "langCode": "ssw" },
+ "st": { "label": "Sesotho", "langCode": "sot" },
+ "su": { "label": "Sundanese", "langCode": "sun" },
+ "sv": { "label": "Swedish", "langCode": "swe" },
+ "sw": { "label": "Swahili", "langCode": "swa" },
+ "syr": { "label": "Syriac", "langCode": "syr" },
+ "ta": { "label": "Tamil", "langCode": "tam" },
+ "te": { "label": "Telugu", "langCode": "tel" },
+ "tg": { "label": "Tajik", "langCode": "tgk" },
+ "th": { "label": "Thai", "langCode": "tha" },
+ "ti": { "label": "Tigrinya", "langCode": "tir" },
+ "tk": { "label": "Turkmen", "langCode": "tuk" },
+ "tl": { "label": "Tagalog", "langCode": "tgl" },
+ "tlh": { "label": "Klingon", "langCode": "tlh" },
+ "tn": { "label": "Tswana", "langCode": "tsn" },
+ "to": { "label": "Tonga", "langCode": "tog" },
+ "tr": { "label": "Turkish", "langCode": "tur" },
+ "ts": { "label": "Tsonga", "langCode": "tso" },
+ "tt": { "label": "Tatar", "langCode": "tat" },
+ "ug": { "label": "Uighur", "langCode": "uig" },
+ "uk": { "label": "Ukrainian", "langCode": "ukr" },
+ "ur": { "label": "Urdu", "langCode": "urd" },
+ "uz": { "label": "Uzbek", "langCode": "uzb" },
+ "ve": { "label": "Venda", "langCode": "ven" },
+ "vi": { "label": "Vietnamese", "langCode": "vie" },
+ "vo": { "label": "Volapuk", "langCode": "vol" },
+ "war": { "label": "Waray Philippines", "langCode": "war" },
+ "wo": { "label": "Wolof", "langCode": "wol" },
+ "xh": { "label": "Xhosa", "langCode": "xho" },
+ "yi": { "label": "Yiddish", "langCode": "yid" },
+ "yo": { "label": "Yoruba", "langCode": "yor" },
+ "za": { "label": "Zhuang", "langCode": "zha" },
+ "zh": { "label": "Chinese Simplified", "langCode": "zho" },
+ "zh-Hant": { "label": "Chinese Traditional", "langCode": "zho" },
+ "zu": { "label": "Zulu", "langCode": "zul" }
+}

packages/client-adapters/src/client-adapters.js

@@ -8,6 +8,7 @@ import ArethusaTreebankAdapter from '@clAdapters/adapters/arethusa/adapter'
 import AlpheiosLogeionAdapter from '@clAdapters/adapters/logeion/adapter'
 import AlpheiosTokenizationAdapter from '@clAdapters/adapters/tokenization/adapter'
 import DTSAPIAdapter from '@clAdapters/adapters/dtsapi/adapter'
+import DetectLangAdapter from '@clAdapters/adapters/detectlang/adapter'
 
 import WrongMethodError from '@clAdapters/errors/wrong-method-error'
 import NoRequiredParamError from '@clAdapters/errors/no-required-param-error'
@@ -93,6 +94,11 @@ class ClientAdapters {
  return cachedAdaptersList.get('dtsapiGroup')
  }
 
+ static get detectlangGroup () {
+ ClientAdapters.init()
+ return cachedAdaptersList.get('detectlangGroup')
+ }
+
  /**
  * This method checks if given method is registered in config for category.adapterName
  * @param {String} category - category name - morphology, lemmatranslation, lexicon
@@ -427,6 +433,10 @@ class ClientAdapters {
  return null
  }
 
+ /**
+ * It is used for getting TEI texts from DTS API
+ * @param {Object} options
+ */
  static async dtsApiMethod (options) {
  ClientAdapters.checkMethodParam('dtsapiGroup', 'dtsapi', options)
 
@@ -453,6 +463,27 @@ class ClientAdapters {
  return { result: res, errors: localDTSAPIAdapter.errors }
  }
  }
+
+ /**
+ * It is used for detecting language by text
+ * @param {Object} options
+ */
+ static async detectLangMethod (options) {
+ ClientAdapters.checkMethodParam('detectlangGroup', 'detectlang', options)
+
+ const localDetectLangAdapter = new DetectLangAdapter({
+ category: 'detectlangGroup',
+ adapterName: 'detectlang',
+ method: options.method,
+ clientId: options.clientId,
+ sourceData: options.params.sourceData
+ })
+
+ if (options.method === 'getDetectedLangsList') {
+ const res = await localDetectLangAdapter.getDetectedLangsList(options.params.text)
+ return { result: res, errors: localDetectLangAdapter.errors }
+ }
+ }
 }
 
 export default ClientAdapters

packages/client-adapters/src/locales/en-us/messages.json

@@ -204,5 +204,16 @@
  "description": "Error message for DTS API adapter - problems with fetching words from api",
  "component": "dtsapiGroup.general",
  "params": ["message"]
+ },
+ "DETECT_LANG_URL_ERROR": {
+ "message": "There are not enough parameters for detect language request",
+ "description": "Error message - no apikey and baseurl for Alpheios Tokenization API",
+ "component": "detectlangGroup.detectlang"
+ },
+ "DETECT_LANG_FETCH_ERROR": {
+ "message": "Some problems with detection language request API - {message}",
+ "description": "Error message for DetectLang adapter",
+ "component": "detectlangGroup.detectlang",
+ "params": ["message"]
  }
 }