PyQ Release 4.0.2

doc/_kx/mkref.py

@@ -10,7 +10,7 @@
 
    {short}{doc}
 
-   See also `{qname} on code.kx.com <http://code.kx.com/wiki/Reference/{qname}>`_.
+   See also `{qname} on code.kx.com <{url}>`_.
 
 
 """
@@ -22,7 +22,7 @@
    {short}
 
    For details, see :func:`q.{name} <pyq.q.{name}>` and `{qname} on code.kx.com
-   <http://code.kx.com/wiki/Reference/{qname}>`_.
+   <{url}>`_.
 
 
 """
@@ -42,7 +42,7 @@
 ref_dir = script_dir / '..' / 'reference'
 q_func_rst = ref_dir / 'q-funcs.rst'
 k_meth_rst = ref_dir / 'K-meths.rst'
-code_dump = script_dir / 'code.dump'
+code_dump = script_dir / 'code.kx.dump'
 kx_docs = code_dump.load()
 
 #  Fix links
@@ -54,8 +54,8 @@
         qname = fixes.get(name, qname)
         desc = kx_docs.get(qname)
         if isinstance(desc, tuple):
-            short = desc[1]
+            _, short, url = desc
         else:
             short = 'The %s function.' % qname
-        f.write(Q_FUNC_DOC.format(name=name, short=short, qname=qname, doc=DOC[name]))
-        g.write(K_METH_DOC.format(name=name, short=short, qname=qname))
+        f.write(Q_FUNC_DOC.format(name=name, short=short, qname=qname, url=url, doc=DOC[name]))
+        g.write(K_METH_DOC.format(name=name, short=short, qname=qname, url=url))

doc/_kx/code.py → doc/_kx/parse.py

@@ -1,12 +1,12 @@
 # Grab function and description from code.kx.com which still uses mediawiki
 # Requires Python 3.6+
 
-from time import sleep
 from pickle import dump
+from urllib.parse import urljoin
 
 import py
 import requests
-
+from bs4 import BeautifulSoup as bs
 
 KWDS = ['abs', 'acos', 'aj', 'aj0', 'all', 'and', 'any', 'asc', 'asin', 'asof',
         'atan', 'attr', 'avg', 'avgs', 'bin', 'binr', 'ceiling', 'cols', 'cor', 'cos',
@@ -27,35 +27,34 @@
         'update', 'upper', 'upsert', 'value', 'var', 'view', 'views', 'vs', 'wavg',
         'where', 'while', 'within', 'wj', 'wj1', 'wsum', 'ww', 'xasc', 'xbar', 'xcol',
         'xcols', 'xdesc', 'xexp', 'xgroup', 'xkey', 'xlog', 'xprev', 'xrank']
-URL_PATTERN = "http://code.kx.com/wiki/Reference/{}?action=raw"
+
 HEADERS = {
     # Let's brake their statistics :)
-    'User-Agent': 'Mozilla/4.0 (compatible; MSIE 4.0; Windows NT)'
+    'User-Agent': 'Mozilla/2.0 (compatible; MSIE 3.0; Windows 3.1)'
 }
 
-OUT = {}
-
-for kwd in KWDS:
-    url = URL_PATTERN.format(kwd)
-    r = requests.get(url, headers=HEADERS)
-    if r.status_code == requests.codes.ok:
-        line = r.text.splitlines()[0]
-        try:
-            _, func, desc = line.replace('}', '').split('|')
-        except ValueError as e:
-            print(f"{kwd}: Error {e} - {line}")
-            OUT[kwd] = line
-        else:
-            OUT[kwd] = (func, desc)
-            print(f"{kwd}: {func}|{desc}")
-    else:
-        print(f"{kwd}: Error {r.status_code} downloading from {url}.")
-        OUT[kwd] = r.status_code
-
-    sleep(0.5)  # Let's not DDoS them
-
-filename = py.path.local(__file__).new(ext='dump')
-with filename.open('wb') as f:
-    dump(OUT, f)
-
-print(f"Saved into {filename}.")
+CARD = 'http://code.kx.com/q/ref/card/'
+
+out = {}
+r = requests.get(CARD, headers=HEADERS)
+if r.status_code == requests.codes.ok:
+    soup = bs(r.text, 'html.parser')
+    for div in soup.find_all('div', {'class': 'md-content'}):
+        for link in div.find_all('a'):
+            key = link.text.strip().split()[0]
+            if 'class' not in link.attrs:
+                if key in KWDS:
+                    href = urljoin(CARD, link.attrs['href'])
+                    title = link.attrs['title'].strip() if 'title' in link.attrs else key
+                    # strip unicode characters
+                    title = title.encode('ascii', 'ignore').decode().strip()
+                    out[key] = (key, title, href)
+    print("Missing:", ', '.join(k for k in KWDS if k not in out))
+    filename = py.path.local('code.kx.dump')
+    with filename.open('wb') as f:
+        dump(out, f)
+
+    print(f"Saved into {filename}.")
+
+else:
+    print(f"ERROR {r.status_code} downloading page from {CARD}.")

doc/conf.py

@@ -18,7 +18,7 @@
 try:
     from pyq import __version__
 except ImportError:
-    __version__ = '4.0.1'
+    __version__ = '4.0.2'
 
 
 # If extensions (or modules to document with autodoc) are in another directory,

doc/install/install.rst

@@ -120,3 +120,5 @@ Install PyQ:
 .. include:: update.rst
 
 .. include:: centos32on64.rst
+
+.. include:: ubuntu.rst

doc/install/ubuntu.rst

@@ -0,0 +1,54 @@
+Installing PyQ on Ubuntu 16.04
+------------------------------
+
+Since Python provided by Ubuntu is statically linked, shared libraries need to be installed before PyQ can be installed.
+
+Python 2
+........
+
+Install shared libraries:
+
+.. code-block:: bash
+
+    $ sudo apt-get install libpython-dev libpython-stdlib python-pip python-virtualenv
+
+
+Create and activate virtual environment:
+
+.. code-block:: bash
+
+    $ python -m virtualenv -p $(which python2) py2
+    $ source py2/bin/activate
+
+
+Install PyQ:
+
+.. code-block:: bash
+
+    (py2) $ pip install -i https://pyq.enlnt.com --no-binary pyq pyq
+
+
+Python 3
+........
+
+Install shared libraries:
+
+.. code-block:: bash
+
+    $ sudo apt-get install libpython3-dev libpython3-stdlib python3-pip python3-virtualenv
+
+
+Create and activate virtual environment:
+
+.. code-block:: bash
+
+    $ python3 -m virtualenv -p $(which python3) py3
+    $ source py3/bin/activate
+
+
+Install PyQ:
+
+.. code-block:: bash
+
+    (py3) $ pip3 install -i https://pyq.enlnt.com --no-binary pyq pyq
+