Updates

Signed-off-by: denim2x <denim2x@cyberdude.com>

README.md

@@ -43,8 +43,8 @@ dialogflow:
 
 redis:
   - host: <host>
-    port: <port>
-    pass: <password>  # optional
+    port: <port>      # optional (default: 6379)
+    auth: <password>  # optional
   - ...
 ```
 - (optional) *\<project root>/account.json* with valid *GCP service account* data.

_bareasgi.py

@@ -4,3 +4,7 @@
 def json_response(data, status=200, headers={}):
   headers = [] # FIXME
   return _bareasgi.json_response(status, headers, data)
+
+def text_response(text, status=200, headers={}):
+  headers = []
+  return _bareasgi.text_response(status, headers, text)

_rom.py

@@ -0,0 +1,26 @@
+from rom import *
+import rom as _rom
+from rom.columns import String as _String
+from rom.util import set_connection_settings as init, get_connection
+
+
+class String(_String):
+  def _to_redis(self, value):
+    if isinstance(value, bytes):
+      return super()._to_redis(value)
+    return value
+
+class _Model:
+  def __getitem__(self, key):
+    value = getattr(self, key)
+    return value.decode() if isinstance(value, bytes) else bytes
+
+Model = (_Model, _rom.Model)
+
+def bgsave():
+  db = get_connection()
+  try:
+    db.bgsave()
+    return True
+  except:
+    return False

app.js

@@ -46,13 +46,11 @@ function get_knowledge() {
 }
 
 function send_message(text='', cb) {
-  request.post('/message').send(text).then(({ body }) => {
-    state.conversation.push(body);
+  request.post('/message').send(text).then(({ text }) => {
+    state.conversation.push({ text });
   }, (e) => { 
     _error('POST', '/message', e);
-    if (cb) {
-      cb();
-    }
+    cb && cb();
   });
 }
 
@@ -110,20 +108,33 @@ rivets.binders['input'] = {
   },
 
   bind: function(el) {
-    $(el).on('input.rivets', this.publish);
+    this._empty = true;
+    $(el).on('input.rv-input', this.publish);
+    this._watch = () => {
+      el.innerHTML = '';
+      if (el.innerHTML == '') {
+        clearInterval(this._watcher);
+      }
+    };
   },
 
   unbind: function(el) {
-    $(el).off('input.rivets');
+    $(el).off('.rv-input');
+    clearInterval(this._watcher)
   },
 
   routine: function(el, value) {
     if (this.state != 'publish') {
+      clearInterval(this._watcher);
       el.innerText = value;
     }
-    state.error = false;  
+    state.error = false;
+    this._empty = value == '';
+    if (this._empty) {
+      this._watcher = setInterval(this._watch, 30);
+    }
     if (this.empty_class) {
-      $(el).toggleClass(this.empty_class, value == '');
+      $(el).toggleClass(this.empty_class, this._empty);
     }
   },
 

dialogflow.py

@@ -4,6 +4,7 @@
 from dialogflow_v2beta1 import SessionsClient, KnowledgeBasesClient, DocumentsClient
 from dialogflow_v2beta1 import types, enums
 from google.api_core.exceptions import InvalidArgument, GoogleAPICallError
+from google.api_core.retry import Retry
 
 from util import realpath
 from config import project_id
@@ -50,6 +51,10 @@ def __init__(self, session_id=uuid4(), language_code='en'):
     self._kb = kb.project_path(project_id)
     self.language_code = language_code
     self.min_confidence = 0.8
+    self._retry = {
+      'retry': Retry(),
+      'timeout': 10
+    }
 
   def __call__(self, text=None, event=None):
     language_code = self.language_code
@@ -61,24 +66,20 @@ def __call__(self, text=None, event=None):
       query_input = types.QueryInput(event=event_input)
     else:
       return None
-    return session.detect_intent(session=self._session, query_input=query_input)
+    return session.detect_intent(session=self._session, query_input=query_input, **self._retry)
 
-  def get_answers(self, text, raw=False, kb=True, sort_key=None, **kw):
+  def get_answers(self, text, kb=True):
     res = self(text=text)
-    filter_fn = kw.get('filter')
-    if hasattr(res.query_result, 'knowledge_answers'):
-      if not kb and res.alternative_query_results:
-        answer = res.alternative_query_results[0]
-        if answer.intent_detection_confidence >= self.min_confidence:
-          return [answer.fulfillment_text]
-        return None
-      answers = [a for a in res.query_result.knowledge_answers.answers]
-      if filter_fn:
-        answers = list(filter(filter_fn, answers))
-      if sort_key:
-        answers.sort(sort_key)
-      return answers if raw else [a.answer for a in answers]
-    return None
+    if not hasattr(res.query_result, 'knowledge_answers'):
+      return
+
+    if not kb and res.alternative_query_results:
+      answer = res.alternative_query_results[0]
+      if answer.action != 'input.unknown' and answer.intent_detection_confidence >= self.min_confidence:
+        return [answer.fulfillment_text]
+      return None
+
+    return res.query_result.knowledge_answers.answers
 
   def event(self, name, raw=False):
     res = self(event=name)

document.py

@@ -1,14 +1,13 @@
 from urllib.error import HTTPError
 from collections import namedtuple
 
-from util import pq, List
+from util import pq, List, Text, OrderedSet
 
 
 _excludes = (
   'Recommended_Readings',
   'See_Also',
   'Residents',
-  'Paraphernalia',
   'Alternate_Reality_Versions'
 )
 
@@ -18,21 +17,24 @@
     'Links_and_References',
     'References',
     'Points_of_Interest',
-    'Links'
+    'Links',
+    'Related'
   ],
   format=':not(#{item})',
   str=''
 )
 
-Fragment = namedtuple('Fragment', ('caption', 'text'))
+Handle = namedtuple('Handle', ('pointer', 'until'))
+Fragment = namedtuple('Fragment', ('handle', 'text'))
 
 def _text(el, to_strip=None):
   if el is None:
     return None
-  return el.text().strip().strip(to_strip).strip()
+  text = el.text() if isinstance(el, pq) else el
+  return text.strip().strip(to_strip).strip()
 
 class Document:
-  def __init__(self, url=None, name=None, quotes=False):
+  def __init__(self, url=None, name=None, quotes=False, prepare=False):
     if name is not None:
       url = url.format(*name.split('|'))
       self.name = name
@@ -45,15 +47,20 @@ def __init__(self, url=None, name=None, quotes=False):
     except HTTPError:
       doc = pq([])
     self.caption = doc.children('head > title').text().split('|', 1)[0].strip()
-    self.site = doc.find('link[rel="search').attr('title').rstrip('(en)').strip()
+    self.site = doc.find('link[rel="search"]').default('title', '').rstrip('(en)').strip()
     self._doc = doc
     self.__content = None
-    self._data = None
+    self.__h2 = None
+    self._fragments = None
+    self._refs = True
     self._quotes = quotes
-    sel = List(['h3, p, ul, ol'])
+    self._isel = 'text, a, b, i, em, strong, span'
+    sel = List([self._isel, 'p, ul, ol'])
     if self._quotes:
       sel.append('.quote')
     self._sel = str(sel)
+    if prepare:
+      iter(self)
 
   def __bool__(self):
     return bool(self._doc)
@@ -62,43 +69,98 @@ def _content(self):
     if self.__content is None:
       content = self._doc.find('.mw-content-text')
       content.find('.noprint, noscript, script, style, link, iframe, embed, video, img, .editsection').remove()
-      content.find('*').remove_attr('style')
       self.__content = content
     return self.__content
 
   def __iter__(self):
     if not self:
+      raise StopIteration
+
+    if self._fragments is not None:
+      yield from self._fragments
+
+    self._fragments = {}
+    content = self._content()
+
+    self._fragment(content.children('.portable-infobox'), name='Summary', until='#toc')
+    h2_list = content.children(f'h2{scrape_excludes} > {scrape_excludes}').closest('h2')
+    for h2 in h2_list.items():
+      self._fragment(h2)
+      for h3 in h2.nextUntil('h2', 'h3').items():
+        self._fragment(h2, h3)
+
+    yield from tuple(self._fragments)
+
+  def _fragment(self, *pointer, name=None, until=None):
+    if not name:
+      name = '/'.join(_text(h) for h in pointer)
+
+    name = f"{self.name}#{_text(name)}"
+    fragment = Fragment(Handle(pointer[-1], until), None)
+    self._fragments[name] = fragment
+
+  def __getitem__(self, name):
+    if name not in self._fragments:
       return
 
-    if self._data is not None:
-      yield from self._data
+    handle, text = self._fragments[name]
+    if text is not None:
+      return text
 
-    self._data = []
     content = self._content()
-    content.find('.reference').remove()
+
+    if self._refs:
+      content.find('.reference').remove()
+      self._refs = False
+
     if self._quotes:
       for quote in content.find('.quote').items():
         author = quote.find('.selflink').closest('b')
         author.closest('dl').remove()
         _quote = quote.find('i')
         _quote.text('"' + _text(_quote, '"\'') + '"')
         author.append('said').prependTo(_quote.closest('dd'))
-
-    h2_list = content.children(f'h2{scrape_excludes} > {scrape_excludes}').closest('h2')
-    for h2 in h2_list.items():
-      self._append(h2.nextUntil('h2, h3', self._sel), h2)
-      for h3 in h2.nextUntil('h2', 'h3'):
-        self._append(h3.nextUntil('h2, h3', self._sel), h2, h3)
-
-  def _append(self, body, *heads):
-    _data = self._data
-    if _data is None or not body:
-      return False
-
-    caption = List((_text(h) for h in heads), str='/')
-    text = List((_text(e) for e in body.items()), False, str='\n')
-    _data.append(Fragment(f"{self.name}#{caption}", str(text)))
-    return True
+
+      self._quotes = None
+
+    pointer, until = handle
+    if not until:
+      until = 'h2, h3'
+
+    if pointer.children('span').is_('#Abilities, #Equipment, #Transportation, #Weapons'):
+      body = pq([])
+      for li in pointer.nextUntil('h2, h3', 'ul').children('li').items():
+        nodes = pq([])
+        for node in li.contents().items(exclude='ul, b > a'):
+          nodes.extend(Text(_text(node).lstrip(': ').rstrip() + ' ') if node.prev().is_('b > a') else node)
+        body.extend(pq('<p>').append(nodes))
+    else:
+      body = pointer.nextUntil(until, self._sel)
+
+    return self._create(name, body)
+
+  def _create(self, name, body):
+    fragment = self._fragments.get(name)
+    if not body:
+      del self._fragments[name]
+      return
+
+    text = List(banned=False, str='\n')
+    span = List(banned=False, str=' ')
+    for node in body.items():
+      if node.is_(self._isel):
+        span.append(_text(node))
+      else:
+        text.extend([str(span), _text(node)])
+        span.clear()
+    text.append(str(span))
+    text_ = str(text)
+    if not text_:
+      del self._fragments[name]
+      return
+
+    self._fragments[name] = Fragment(fragment.handle, text_)
+    return text_
 
   @staticmethod
   def parse_name(name):