Merge pull request #1172 from UAlbertaALTLab/blackfoot-setup

Deploy dev dictionary for blackfoot, and merge most recent changes by Katie into the blackfoot dictionary.  Future work on the blackfoot dictionary should be done on the main branch now, so it will deploy to the ipowahsin.altlab.dev URL.

.github/workflows/test-and-deploy.yml

@@ -183,7 +183,7 @@ jobs:
       - integration-test
       - build-docker-image
 
-    if: needs.should-deploy.outputs.should-run == 'true'
+    if: needs.should-deploy.outputs.should-run == 'true' && false # Deactivate this task to have successfull outcomes until re-deploying decision
 
     steps:
       - name: Login to GitHub Container Registry

Procfile

@@ -14,3 +14,5 @@ cwdeng: ./cwdeng-manage runserver
 srseng: ./srseng-manage runserver
 hdneng: ./hdneng-manage runserver
 lacombe:  ./crkLacombeeng-manage runserver
+blaeng: ./blaeng-manage runserver
+

blaeng-manage

@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+"""
+Command-line utility for administrative tasks.
+"""
+
+import os
+import sys
+from pathlib import Path
+
+# sys.path[0] is initialized to the directory containing the script, which
+# isn’t right for our purposes.
+sys.path[0] = os.fspath(Path(sys.path[0]) / "src")
+
+if __name__ == "__main__":
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "blaeng.site.settings")
+
+    from django.core.management import execute_from_command_line
+
+    execute_from_command_line(sys.argv)

docker/docker-compose.staging-override.yml

@@ -110,3 +110,19 @@ services:
       - "../src/crkLacombeeng/db/:/app/src/crkLacombeeng/db/"
       - "../src/crkLacombeeng/CreeDictionary/search_quality/:/app/src/crkLacombeeng/CreeDictionary/search_quality/"
 
+  blaeng:
+    # In prod, we *always* want the apps running, including on boot; that’s
+    # not necessarily the case on our dev machines.
+    restart: "no"
+    build:
+      context: ..
+      dockerfile: docker/Dockerfile
+    # The tag that will automatically be applied on build:
+    image: morphodict:dev
+
+    volumes:
+      - "./.env:/app/.env"
+      - "../src/blaeng/resources/vector_models/:/app/src/blaeng/resources/vector_models/"
+      - "../src/blaeng/db/:/app/src/blaeng/db/"
+      - "../src/blaeng/CreeDictionary/search_quality/:/app/src/blaeng/CreeDictionary/search_quality/"
+

docker/docker-compose.yml

@@ -143,3 +143,26 @@ services:
       - "/data_local/application-data/crkLacombeeng/vector_models:/app/src/crkLacombeeng/resources/vector_models/"
       - "/data_local/application-data/crkLacombeeng/db:/app/src/crkLacombeeng/db/"
       - "/data_local/application-data/crkLacombeeng/search_quality:/app/src/crkLacombeeng/CreeDictionary/search_quality/"
+  blaeng:
+    restart: always
+    # Specify hostname to make it clearer which container is active when
+    # using docker to get a shell in the container.
+    hostname: blaeng-container
+    image: ghcr.io/ualbertaaltlab/itwewina.altlab.app:latest
+    command: /app/app.sh blaeng
+    ports:
+      - "8017:8000" # http
+      - "9017:9191" # uWSGI stats
+    volumes:
+      - "./.env:/app/.env"
+
+      # LFS mounts
+      - "../src/morphodict/lexicon/resources/vector_models/:/app/src/morphodict/lexicon/resources/vector_models/:ro"
+      - "../src/blaeng/resources/fst:/app/src/blaeng/resources/fst:ro"
+      - "../src/CreeDictionary/res/fst/:/app/src/CreeDictionary/res/fst/:ro"
+      - "../src/CreeDictionary/res/:/app/src/CreeDictionary/res/:ro"
+      - "../src/blaeng/resources/dictionary/:/app/src/blaeng/resources/dictionary/:ro"
+
+      - "/data_local/application-data/blaeng/vector_models:/app/src/blaeng/resources/vector_models/"
+      - "/data_local/application-data/blaeng/db:/app/src/blaeng/db/"
+      - "/data_local/application-data/blaeng/search_quality:/app/src/blaeng/CreeDictionary/search_quality/"

docker/helpers/settings.py

@@ -16,6 +16,7 @@
     "arpeng": {"port": 8014, "uwsgi_stats_port": 9014},
     "hdneng": {"port": 8015, "uwsgi_stats_port": 9015},
     "crkLacombeeng": {"port": 8016, "uwsgi_stats_port": 9016},
+    "blaeng": {"port": 8017, "uwsgi_stats_port":9017},
 }
 
 APPS = [App(k, **v) for k, v in APP_INFO.items()]

docker/plays/vars.yml

@@ -13,3 +13,4 @@ morphodict_prod_data_dirs:
   - "/data_local/application-data/arpeng"
   - "/data_local/application-data/hdneng"
   - "/data_local/application-data/crkLacombeeng"
+  - "/data_local/application-data/blaeng"

docs/developers-guide.md

@@ -221,6 +221,7 @@ Then you can access the dictionary applications at various port numbers:
  - arpeng: <http://127.0.0.1:8007/>
  - cwdeng: <http://127.0.0.1:8005/>
  - srseng: <http://127.0.0.1:8009/>
+ - blaeng: <http://127.0.0.1:8011/>
 
 Because [cookies are not port-specific for historical insecurity
 reasons](https://stackoverflow.com/questions/1612177/are-http-cookies-port-specific),
@@ -232,6 +233,7 @@ adding the following to `/etc/hosts`:
     127.0.0.1 cwdeng-local
     127.0.0.1 crkeng-local
     127.0.0.1 srseng-local
+    127.0.0.1 blaeng-local
 
 Then you can access the sites with cookie isolation at
 <http://crkeng-local:8000/>, <http://cwdeng-local:8005/>,

scripts/dev-bootstrap

@@ -20,6 +20,7 @@ for LANG_PAIR in \
     crkeng \
     cwdeng \
     srseng \
+    blaeng \
 ; do
     for USE_TEST_DB in true false; do
         export USE_TEST_DB

src/blaeng/README.md

@@ -0,0 +1,64 @@
+# Generating a new site from scratch
+
+These are all the instructions I followed to create this new version.
+
+```
+./crkeng-manage newdictsite --port 8011 -v 2 bla eng
+# Add blaeng to morphodict/src/conftest.py
+# Add MORPHODICT_LANGUAGE_ENDONYM to src/blaeng/site/settings.py
+# Add many other details, copying from Woods Cree settings.py, including:
+# MD_SOURCE_LANGUAGE_NAME MD_SOURCE_LANGUAGE_SHORT_NAME MD_ORTHOGRAPHY MD_DICTIONARY_NAME
+./blaeng-manage migrate
+./blaeng-manage ensurecypressadminuser --superuser
+./blaeng-manage ensuretestdb
+# MISSING TRANSDUCERS 
+
+```
+
+I had to make transducers. Generating from the instructions we just collected today!
+
+`git clone giellalt/lang-bla`
+after generating the FSTs with the default infrastructure:
+```
+hfst-xfst
+```
+And ran:
+```
+read lexc src/fst/morphology/lexicon.lexc
+define Morphology
+source src/fst/bla-phonology.xfscript
+define Phonology
+regex ~[ $[ "+Err/Frag" ]];
+define removeFragments
+
+regex ~[ $[ "+Err/Orth" ]];
+define removeNonStandardForms
+regex $[ "+N" | "+V" | "+Ipc" | "+Pron" ];
+define selectDictPOS
+set flag-is-epsilon ON
+regex [ selectDictPOS .o. removeNonStandardForms .o. removeFragments .o. Morphology .o. Phonology ];
+save stack generator-gt-dict-norm.hfst
+define NormativeGenerator
+regex [ [ "<" | ">" | "/" ] -> 0 ];
+define removeBoundaries
+load src/fst/orthography/spellrelax.compose.hfst
+define SpellRelax
+regex [ selectDictPOS .o. removeFragments .o. Morphology .o. Phonology .o. removeBoundaries .o. SpellRelax ];
+# regex [ NormativeGenerator .o. removeBoundaries .o. SpellRelax ];
+invert net
+save stack analyser-gt-dict-desc.hfst
+define DescriptiveAnalyser
+```
+
+
+And then we create the `hfstol` files with:
+
+```
+hfst-fst2fst -O -i INPUT.hfst -o OUTPUT.hfstol
+```
+
+After this, `./blaeng-manage ensuretestdb` works.
+
+```
+./blaeng-manage importjsondict src/blaeng/resources/dictionary/blaeng_test_db.importjson
+```