Added async feature, added more info to readme

config.yaml.example

@@ -15,3 +15,6 @@ get:
   - https://<USER:PW>@gondul.tg23.gathering.org/api/public/ping
   - https://<USER:PW>@gondul.tg23.gathering.org/api/public/switches
   - https://<USER:PW>@gondul.tg23.gathering.org/api/public/switch-state
+  - 'https://netbox.tg23.gathering.org/api/ipam/ip-addresses?exclude=config_context&format=json&limit=0':
+      headers:
+        Authorization: 'Token <TOKEN>'

readme.md

@@ -1,10 +1,14 @@
 # Templating Service
 
-THIS HAS NOT BEEN TESTED VERY MUCH, YET :) #alpha/beta/something
-
 This is a new version of our templating engine!
 
-There's one weird thing, to be a bit backwards compatible the last two items of a path will be used as the objects name.
+Originally developed by [@KristianLyng](https://github.com/KristianLyng), later slightly rewritten by [@Foxboron](https://github.com/Foxboron).
+
+Old source can still be found here: [gondul/templating](https://github.com/gathering/gondul/commits/master/templating/templating.py)
+
+## Caveats
+
+There's one weird thing. To be a bit backwards compatible the last two items of a path will be used as the objects name.
 
 This however opens up for possible name collisions from files and http api data.
 
@@ -22,12 +26,16 @@ get:
 All of these would be added as `read/networks` in the python dicts. Aka, the last one is going to win.
 Therefor, think twice when naming things :)
 
+This is inteded behaviour to support as of now. Helps us run templating offline.
+
 ## Features
 
+- Fully async
 - Loads data from local files
   - yaml
   - json
 - Loads data from API's
+- Run a template once with HTTP options
 
 ## Currently supported URI locations
 
@@ -37,6 +45,7 @@ Therefor, think twice when naming things :)
   - Can be relative path
 - `https://` and `http://`
   - Gets data from API endpoint
+  - Supports adding header options (tbh, i just **kwargs this directly to aiohttp..)
 
 ## Example file
 
@@ -48,14 +57,25 @@ get:
   - file://./templating/data
   - https://<USER:PW>@gondul.tg23.gathering.org/api/read/networks
   - http://gondul.tg23.gathering.org/api/public/switches
+ - 'https://netbox.tg23.gathering.org/api/ipam/ip-addresses?exclude=config_context&format=json&limit=0':
+      headers:
+        Authorization: 'Token <TOKEN>'
 ```
 
 ## Example Execute
 
+### Start server on localhost:8080
+
 ```bash
 python3 templating.py -t ../tech-templates/ -c config.yaml
 ```
 
+### Run a template once with options
+
+```bash
+python3 ../templating/templating.py --once magic.conf -c templating/config.yml -t . -i switch=e7-4
+```
+
 ## Mitigate SSTI Vulnerability
 
 This service exposes a api endpoit where you can POST a template and have it rendered.

requirements.txt

@@ -1,3 +1,6 @@
 netaddr
 requests
-flask
+flask
+pynetbox
+aiohttp
+aiofiles

templating.py

@@ -6,9 +6,15 @@
 import os
 import errno
 
+import aiohttp
+import asyncio
+
 import netaddr
 import requests
 
+import time
+import aiofiles
+
 from pathlib import Path
 
 import yaml
@@ -34,53 +40,111 @@ def is_yaml(s):
 def createObjName(i):
     return str(Path(*Path(os.path.splitext(i)[0]).parts[-2:]))
 
-def loadFile(file):
-    if is_yaml:
-        pass
+def parseFile(i):
+    path = i.netloc + i.path
+    if not os.path.isabs(path):
+        path = os.path.normpath(os.getcwd() + i.path)
+    if os.path.isdir(path):
+        for subdir, dirs, files in os.walk(path):
+              for file in files:
+                   return os.path.join(subdir, file)
+    elif os.path.isfile(path):
+        return path
     else:
-        raise ValueError(f'Failed to load {file}. Not a JSON or YAML formatted file')
-    with open(file, 'r') as f:
-        d = yaml.load(f.read(), Loader=yaml.CSafeLoader)
-        objects[createObjName(file)] = d
-
-def getEndpoint(uri: str) -> dict:
-    """
-    Fetches an endpoint and returns the data as a dict.
-    """
-    r = requests.get(uri, timeout=args.timeout)
-    r.raise_for_status()
-    return r.json()
-
-def loadUri(u):
-    a = '/'.join(u.path.split('/')[-2:])
-    objects[a] = getEndpoint(u.geturl())
-
-def load(i):
-    if i.scheme == "file":
-        path = i.netloc + i.path
-        if not os.path.isabs(path):
-            path = os.path.normpath(os.getcwd() + i.path)
-        if os.path.isdir(path):
-            for subdir, dirs, files in os.walk(path):
-                    for file in files:
-                        loadFile(os.path.join(subdir, file))
-        elif os.path.isfile(path):
-            loadFile(path)
-        else:
-            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path)
-    elif i.scheme == "https" or i.scheme == "http":
-        loadUri(i)
-
+        raise FileNotFoundError(
+            errno.ENOENT, os.strerror(errno.ENOENT), path)
 
 def load_conf_file(config_file):
    with open(config_file, "r") as f:
        config = yaml.load(f.read(), Loader=yaml.CSafeLoader)
    return config
 
+
+async def getEndpoint(session, url, **options):
+    async with session.get(url, timeout=args.timeout, **options) as resp:
+        try:
+            resp_body = await resp.json(content_type=None)
+        except:
+            resp.raise_for_status()
+        return url, resp_body
+
+
+async def loadUrls(urls):
+    async with aiohttp.ClientSession() as session:
+
+        tasks = []
+        for i in urls:
+            options = dict(list(i.values())[0])
+            url = list(i.keys())[0]
+            tasks.append(asyncio.ensure_future(getEndpoint(session, url, **options)))
+
+        responses = await asyncio.gather(*tasks)
+        for response in responses:
+            key = '/'.join(urlparse(response[0]).path.split('/')[-2:])
+            objects[key] = response[1]
+    return 1
+
+
+async def readFile(file):
+    async with aiofiles.open(file, mode='r') as f:
+        content = await f.read()
+        out = yaml.load(content, Loader=yaml.CSafeLoader)
+        return file, out
+
+async def loadFiles(files):
+    tasks = []
+    for file in files:
+        options = list(file.values())[0]
+        file = list(file.keys())[0]
+        if is_yaml:
+            pass
+        else:
+            raise ValueError(
+                f'Failed to load {file}. Not a JSON or YAML formatted file')
+        tasks.append(asyncio.ensure_future(readFile(file)))
+
+    contents = await asyncio.gather(*tasks)
+    for content in contents:
+        objects[createObjName(content[0])] = content[1]
+    return 1
+
+
+async def runTasks(tasks):
+    await asyncio.gather(*tasks)
+
+
 def updateData():
+
     config = load_conf_file(args.config)
-    for i in config["get"]:
-        load(urlparse(i))
+
+    files = []
+    urls = []
+    for item in config["get"]:
+        if isinstance(item, dict):
+            pass
+        elif isinstance(item, str):
+            item = {item: ''}
+        else:
+            sys.exit(f"{item} is not a str, or dict. Exit")
+
+        options = list(item.values())[0]
+        item = list(item.keys())[0]
+        item = urlparse(item)
+
+        if item.scheme == "file":
+            d = {parseFile(item): options}
+            files.append(d)
+        elif item.scheme == "https" or item.scheme == "http":
+            item = item.geturl()
+            d = {item: options}
+            urls.append(d)
+
+
+    tasks = []
+    tasks.append(loadUrls(urls))
+    tasks.append(loadFiles(files))
+    asyncio.run(runTasks(tasks))
+
 
 env = Environment(extensions=['jinja2.ext.do'], loader=FileSystemLoader([]), trim_blocks=True, lstrip_blocks=True)
 
@@ -117,19 +181,18 @@ def render_template(tpl, options):
         updateData()
         template = env.get_template(tpl)
         body = template.render(objects=objects, options=options)
-    except (requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError) as err:
-        return f'Timeout or connection error from gondul: {err}', 500
+    except (aiohttp.client_exceptions.ClientConnectorError, aiohttp.client_exceptions.ClientResponseError) as err:
+        return f'Connection error trying to get: {err}', 500
     except TemplateNotFound:
         return f'Template "{tpl}" not found\n', 404
     except TemplateError as err:
         return f'Templating of "{tpl}" failed to render. Most likely due to an error in the template. Error transcript:\n\n{err}\n----\n\n{traceback.format_exc()}\n', 400
-    except requests.exceptions.HTTPError as err:
-        return f'HTTP error from gondul: {err}', 500
     except FileNotFoundError as err:
         return f'File error: {err}', 500
     except ValueError as err:
         return f'Parsing Error: {err}', 500
     except Exception as err:
+        print(traceback.format_exc())
         return f'Uncaught error: {err}', 500
     return body, 200
 
@@ -157,18 +220,26 @@ def root_post(path):
 parser.add_argument("-p", "--port", type=int, default=8080, help="host port")
 parser.add_argument("--debug", action="store_true",
                     help="enable debug mode")
-parser.add_argument("-x", "--timeout", type=int, default=2,
+parser.add_argument("-x", "--timeout", type=int, default=20,
                     help="gondul server timeout")
 parser.add_argument("-o", "--once", type=str, default="",
                     help="Run once with the provided template")
 parser.add_argument("-i", "--options", type=str, default="", nargs="+",
                     help="Options to send to template, like query params in the API")
 parser.add_argument("-f", "--outfile", type=str, default="",
                     help="Output file, otherwise prints to stdout")
+parser.add_argument("--help",  type=str, default="",
+                    help="Print help")
+
+try:
+    args = parser.parse_args()
+except:
+    parser.print_help()
+    sys.exit(0)
 
-args = parser.parse_args()
 env.loader.searchpath = args.templates
 
+
 if not sys.argv[1:]:
     parser.print_help()
     sys.exit(1)