[FEATURE-REQUEST]: File Handler and Storage Service

docs/main_readme.md

@@ -92,6 +92,7 @@
  - [Stripe](docs/docs/development/std_actions/9_stripe.md)
  - [Vectors](docs/docs/development/std_actions/10_vectors.md)
  - [Elastic](docs/docs/development/std_actions/11_elastic.md)
+ - [File Handler](docs/docs/development/std_actions/12_file_handler.md)
  - [Jaseci Core](docs/docs/development/std_actions/1_jaseci.md)
  - [Alias](docs/docs/development/std_actions/1_jaseci.md#alias)
  - [Objects](docs/docs/development/std_actions/1_jaseci.md#objects)

jaseci_core/jaseci/__init__.py

@@ -17,6 +17,8 @@ def load_standard():
  import jaseci.extens.act_lib.request # noqa
  import jaseci.extens.act_lib.std # noqa
  import jaseci.extens.act_lib.file # noqa
+ import jaseci.extens.act_lib.file_handler # noqa
+ import jaseci.extens.act_lib.storage # noqa
  import jaseci.extens.act_lib.vector # noqa
  import jaseci.extens.act_lib.date # noqa
  import jaseci.extens.act_lib.jaseci # noqa

jaseci_core/jaseci/cli_tools/jsctl.py

@@ -15,6 +15,7 @@
 from jaseci.prim.super_master import SuperMaster
 from jaseci.utils.utils import copy_func
 from .book_tools import Book, modifiedBook
+from jaseci.utils.file_handler import FileHandler
 from jaseci.utils.utils import logger, perf_test_start, perf_test_stop, find_first_api
 from jaseci.jsorc.jsorc import JsOrc
 from prettytable import PrettyTable
@@ -102,6 +103,12 @@ def remote_api_call(payload, api_name):
  )
  if ret.status_code > 205:
  ret = f"Status Code Error {ret.status_code}\n{ret.json()}"
+ elif ret.headers.get("Content-Type", None) == "application/octet-stream":
+ file_handler = FileHandler.fromRequest(
+ ret.content, ret.headers.get("Content-Disposition")
+ )
+ session["master"]._h.add_file_handler(file_handler)
+ ret = file_handler.attr()
  else:
  ret = ret.json()
  return ret
@@ -172,12 +179,12 @@ def interface_api(api_name, is_public, is_cli_only, **kwargs):
  else:
  out = session["master"].general_interface_to_api(kwargs, api_name)
  d_out = out
- if (
- isinstance(out, dict)
- and "report_custom" in out.keys()
- and out["report_custom"] is not None
- ):
- out = out["report_custom"]
+ if isinstance(out, dict):
+ if "report_custom" in out.keys() and out["report_custom"] is not None:
+  out = out["report_custom"]
+ elif "report_file" in out.keys() and out["report_file"] is not None:
+  out = session["master"]._h.get_file_handler(out["report_file"]).attr()
+
  if isinstance(out, dict) or isinstance(out, list):
  out = json.dumps(out, indent=2)
  click.echo(out)

jaseci_core/jaseci/extens/act_lib/file_handler.py

@@ -0,0 +1,183 @@
+"""Built in actions for Jaseci"""
+import mimetypes
+
+from requests import get
+from jaseci.jsorc.live_actions import jaseci_action
+
+
+@jaseci_action(act_group=["fh"])
+def load(path: str, meta: dict = {}):
+ """temp"""
+ from jaseci.utils.file_handler import FileHandler
+
+ return meta["h"].add_file_handler(FileHandler.fromPath(path))
+
+
+@jaseci_action(act_group=["fh"])
+def new(
+ name: str,
+ content_type: str = None,
+ field: str = None,
+ persist: bool = False,
+ meta: dict = {},
+):
+ """temp"""
+ from jaseci.utils.file_handler import FileHandler
+
+ return meta["h"].add_file_handler(
+ FileHandler(name=name, content_type=content_type, field=field, persist=persist)
+ )
+
+
+@jaseci_action
+def guess_type(filename: str) -> tuple:
+ """temp"""
+ return mimetypes.guess_type(filename)
+
+
+@jaseci_action(act_group=["fh"])
+def update(
+ id: str,
+ name: str = None,
+ content_type: str = None,
+ field: str = None,
+ persist: bool = None,
+ meta: dict = {},
+):
+ """temp"""
+ from jaseci.utils.file_handler import FileHandler
+
+ file_handler: FileHandler = meta["h"].get_file_handler(id)
+ if name:
+ file_handler.name = name
+ if not content_type:
+ file_handler.content_type = mimetypes.guess_type(name)[0]
+
+ if content_type:
+ file_handler.content_type = content_type
+
+ if field:
+ file_handler.field = field
+
+ if persist != None:
+ file_handler.persist = persist
+
+ return file_handler.attr()
+
+
+@jaseci_action(act_group=["fh"])
+def read(id: str, offset: int = None, meta: dict = {}):
+ """temp"""
+ return meta["h"].get_file_handler(id).read(offset)
+
+
+@jaseci_action(act_group=["fh"])
+def seek(id: str, offset: int, whence: int = 0, meta: dict = {}):
+ """temp"""
+ return meta["h"].get_file_handler(id).seek(offset, whence)
+
+
+@jaseci_action(act_group=["fh"])
+def open(id: str, mode: str = "r", encoding: str = "utf-8", meta: dict = {}, **kwargs):
+ """temp"""
+ meta["h"].get_file_handler(id).open(mode, encoding, False, **kwargs)
+
+
+@jaseci_action(act_group=["fh"])
+def is_open(id: str, meta: dict = {}):
+ """temp"""
+ return meta["h"].get_file_handler(id).is_open()
+
+
+@jaseci_action(act_group=["fh"])
+def exists(id: str, meta: dict = {}):
+ """temp"""
+ return meta["h"].get_file_handler(id).exists()
+
+
+@jaseci_action(act_group=["fh"])
+def write(id: str, content: str, meta: dict = {}):
+ """temp"""
+ meta["h"].get_file_handler(id).write(content)
+
+
+@jaseci_action(act_group=["fh"])
+def flush(id: str, meta: dict = {}):
+ """temp"""
+ meta["h"].get_file_handler(id).flush()
+
+
+@jaseci_action(act_group=["fh"])
+def close(id: str, meta: dict = {}):
+ """temp"""
+ meta["h"].get_file_handler(id).close()
+
+
+@jaseci_action(act_group=["fh"])
+def detach(id: str, persist: bool = None, meta: dict = {}):
+ """temp"""
+ from jaseci.utils.file_handler import FileHandler
+
+ file_handler: FileHandler = meta["h"].pop_file_handler(id)
+ file_info = file_handler.attr()
+
+ if (persist is None and not file_handler.persist) or persist is False:
+ file_handler.delete()
+ else:
+ file_handler.close()
+
+ return file_info
+
+
+@jaseci_action(act_group=["fh"])
+def delete(id: str, meta: dict = {}):
+ """temp"""
+ meta["h"].pop_file_handler(id).delete()
+
+
+@jaseci_action(act_group=["fh"])
+def attr(id: str, meta: dict = {}):
+ """temp"""
+ return meta["h"].get_file_handler(id).attr()
+
+
+@jaseci_action(act_group=["fh"])
+def to_bytes(id: str, meta: dict = {}):
+ """temp"""
+ return meta["h"].get_file_handler(id).to_bytes()
+
+
+@jaseci_action(act_group=["fh"])
+def to_base64(id: str, meta: dict = {}):
+ """temp"""
+ return meta["h"].get_file_handler(id).to_base64()
+
+
+@jaseci_action(act_group=["fh"])
+def to_json(id: str, meta: dict = {}):
+ """temp"""
+ return meta["h"].get_file_handler(id).to_json()
+
+
+@jaseci_action(act_group=["fh"])
+def dump_json(id: str, json, indent: int = None, meta: dict = {}):
+ """temp"""
+ meta["h"].get_file_handler(id).dump_json(json, indent)
+
+
+@jaseci_action(act_group=["fh"])
+def download(url: str, header: dict = {}, meta: dict = {}):
+ """Standard built in for download file from url"""
+ from jaseci.utils.file_handler import FileHandler
+
+ tmp = FileHandler("tmp")
+ meta["h"].add_file_handler(tmp)
+
+ with get(url, stream=True, headers=header) as res:
+ res.raise_for_status()
+ tmp.open(mode="wb", encoding=None)
+ for chunk in res.iter_content(chunk_size=8192):
+ tmp.buffer.write(chunk)
+ tmp.close()
+
+ return tmp.id

jaseci_core/jaseci/extens/act_lib/request.py

@@ -1,8 +1,6 @@
 """Built in actions for Jaseci"""
 import requests
 from jaseci.jsorc.live_actions import jaseci_action
-from base64 import b64decode, b64encode
-from io import BytesIO
 
 
 @jaseci_action()
@@ -120,7 +118,7 @@ def options(url: str, data: dict, header: dict):
 
 
 @jaseci_action()
-def multipart_base64(url: str, files: list, header: dict):
+def multipart(url: str, files: list, header: dict, meta: dict):
  """
  Issue request
  Param 1 - url
@@ -132,6 +130,8 @@ def multipart_base64(url: str, files: list, header: dict):
  Return - response object
  """
 
+ hook = meta["h"]
+
  if not files:
  return {
  "status_code": 400,
@@ -142,12 +142,17 @@ def multipart_base64(url: str, files: list, header: dict):
 
  form_data = []
 
+ stream_to_be_close = []
+
  if files is not None:
  for f in files:
+ file_handler = hook.get_file_handler(f)
+ stream = file_handler.open("rb", None, True)
+ stream_to_be_close.append(stream)
  form_data.append(
  (
- f["field"] if "field" in f else "file",
- (f["name"], BytesIO(b64decode(f["base64"]))),
+ file_handler.field or "file",
+ (file_handler.name, stream, file_handler.content_type),
  )
  )
 
@@ -157,16 +162,8 @@ def multipart_base64(url: str, files: list, header: dict):
  ret["response"] = res.json()
  except Exception:
  ret["response"] = res.text
- return ret
 
+ for stream in stream_to_be_close:
+ stream.close()
 
-@jaseci_action()
-def file_download_base64(url: str, header: dict, encoding: str = "utf-8"):
- """Standard built in for download file from url"""
- with requests.get(url, stream=True, headers=header) as res:
- res.raise_for_status()
- with BytesIO() as buffer:
- for chunk in res.iter_content(chunk_size=8192):
- buffer.write(chunk)
- ret = buffer.getvalue()
- return b64encode(ret).decode(encoding)
+ return ret

jaseci_core/jaseci/extens/act_lib/storage.py

@@ -0,0 +1,47 @@
+from jaseci.jsorc.live_actions import jaseci_action
+from jaseci.jsorc.jsorc import JsOrc
+from jaseci.extens.svc.storage_svc import StorageService
+
+
+def storage():
+ return JsOrc.svc("store", StorageService)
+
+
+@jaseci_action()
+def upload(file: str, provider: str = None, container: str = None, meta: dict = {}):
+ """temp"""
+ from jaseci.utils.file_handler import FileHandler
+
+ file_handler: FileHandler = meta["h"].get_file_handler(file)
+
+ return storage().upload(file=file_handler, provider=provider, container=container)
+
+
+@jaseci_action()
+def download(file: str, provider: str = None, container: str = None, meta: dict = {}):
+ """temp"""
+ return meta["h"].add_file_handler(storage().download(file, provider, container))
+
+
+@jaseci_action()
+def delete(file: str, provider: str = None, container: str = None):
+ """temp"""
+ return storage().delete(file, provider, container)
+
+
+@jaseci_action()
+def create_container(name: str, provider: str = None):
+ """temp"""
+ return storage().create_container(name, provider) != None
+
+
+@jaseci_action()
+def has_container(name: str, provider: str = None):
+ """temp"""
+ return storage().has_container(name, provider)
+
+
+@jaseci_action()
+def delete_container(name: str, provider: str = None):
+ """temp"""
+ return storage().delete_container(name, provider)