Test change to definition vectors: May need future reverting but star…

…ts a conversation.

src/morphodict/cvd/__init__.py

@@ -7,6 +7,7 @@
 from gensim.models import KeyedVectors
 
 from morphodict.lexicon import MORPHODICT_LEXICON_RESOURCE_DIR
+from morphodict.relabelling import LABELS
 
 logger = logging.getLogger(__name__)
 
@@ -74,6 +75,9 @@ def uniq(l: list) -> list:
     return list(dict.fromkeys(l))
 
 
+# Note: The computation of this vector for definition is memoized by the builddefinitionvectors django command.
+# They are not live-computed.  If there are changes to this code, please make sure to re-run builddefinitionvectors
+# as a command.
 def vector_for_keys(keyed_vectors, keys: list[str]):
     """Return the sum of vectors in keyed_vectors for the given keys"""
     if not keys:
@@ -85,7 +89,7 @@ def vector_for_keys(keyed_vectors, keys: list[str]):
 RE_PUNCTUATION = re.compile(r'[!,.\[\]\(\)\{\};:"/\?]+')
 
 
-def extract_keyed_words(query: str, keys, already_warned=None):
+def extract_keyed_words(query: str, keys, already_warned=None, analysis=[]):
     """Split query into a list of words that occur in keys
 
     already_warned is an optional set, used to reduce debug log verbosity
@@ -112,6 +116,12 @@ def extract_keyed_words(query: str, keys, already_warned=None):
                     _warn(piece, f"not found: {word!r} piece {piece!r}", already_warned)
         else:
             _warn(word, f"not found: {word!r}", already_warned)
+    analysis = LABELS.linguistic_short.get_longest(
+        tuple([t.strip("+") for t in analysis])
+    )
+
+    if analysis:
+        ret.extend(extract_keyed_words(analysis, keys, already_warned))
 
     return uniq(ret)
 

src/morphodict/cvd/management/commands/builddefinitionvectors.py

@@ -47,7 +47,12 @@ def handle(self, output_file, debug_output_file, **options):
         with create_debug_output(debug_output_file) as debug_output:
             for d in tqdm(definitions.iterator(), total=count):
                 keys = extract_keyed_words(
-                    d.semantic_definition, news_vectors, unknown_words
+                    d.semantic_definition,
+                    news_vectors,
+                    unknown_words,
+                    analysis=(
+                        d.wordform.raw_analysis[2] if d.wordform.raw_analysis else []
+                    ),
                 )
                 debug_output(
                     json.dumps(

src/morphodict/cvd/news-vectors.ipynb

@@ -1214,7 +1214,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -1228,7 +1228,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.4"
+   "version": "3.10.15"
   }
  },
  "nbformat": 4,