diff --git a/.gitignore b/.gitignore index c303cbf..9783d61 100644 --- a/.gitignore +++ b/.gitignore @@ -324,3 +324,5 @@ dist /.ijwb/ /.aswb/ /.clwb/ + +.secretnote diff --git a/packages/secretnote/src/modules/server/server-manager.ts b/packages/secretnote/src/modules/server/server-manager.ts index f6aa2b4..3ff3da7 100644 --- a/packages/secretnote/src/modules/server/server-manager.ts +++ b/packages/secretnote/src/modules/server/server-manager.ts @@ -1,8 +1,6 @@ import { ServerConnection, URL } from '@difizen/libro-jupyter'; import { Emitter, inject, prop, singleton } from '@difizen/mana-app'; -import { uuid } from '@/utils'; - import type { IServer } from './protocol'; import { ServerStatus } from './protocol'; @@ -30,6 +28,11 @@ export class SecretNoteServerManager { if (response.status === 200) { const data = (await response.json()) as IServer[]; for (const item of data) { + /** + * For historical reasons, the server id front-end uses a string and the server returns an int, + * preserving the distinction for now + */ + item.id = item.id.toString(); const spec = await this.getServerSpec(item); if (spec) { item.status = ServerStatus.running; @@ -49,7 +52,6 @@ export class SecretNoteServerManager { async addServer(server: Partial) { const newServer = { - id: uuid(), name: server.name || 'Someone', address: server.address || '', status: ServerStatus.closed, @@ -62,7 +64,6 @@ export class SecretNoteServerManager { const init = { method: 'POST', body: JSON.stringify({ - id: newServer.id, name: newServer.name, address: newServer.address, }), @@ -71,8 +72,13 @@ export class SecretNoteServerManager { if (response.status === 200) { newServer.status = ServerStatus.running; newServer.kernelspec = spec; - this.servers.push(newServer); - this.onServerAddedEmitter.fire(newServer); + const data = await response.json(); + const added = { + ...newServer, + id: data.id, + }; + this.servers.push(added); + this.onServerAddedEmitter.fire(added); return newServer; } } catch (e) { @@ -147,14 +153,14 @@ export class SecretNoteServerManager { } } - getServerSettings(server: IServer) { + getServerSettings(server: Partial) { return { baseUrl: `http://${server.address}/`, wsUrl: `ws://${server.address}/`, }; } - private async getServerSpec(server: IServer) { + private async getServerSpec(server: Partial) { const settings = { ...this.serverConnection.settings, ...this.getServerSettings(server), diff --git a/pyprojects/secretnote/.jupyter/config_dev.py b/pyprojects/secretnote/.jupyter/config_dev.py index ee67a3a..2b51f72 100644 --- a/pyprojects/secretnote/.jupyter/config_dev.py +++ b/pyprojects/secretnote/.jupyter/config_dev.py @@ -1,4 +1,3 @@ -import subprocess from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -18,11 +17,7 @@ c.ServerApp.token = "" -c.ServerApp.root_dir = subprocess.run( - ["git", "rev-parse", "--show-toplevel"], - capture_output=True, - text=True, -).stdout.strip() +c.ServerApp.root_dir = "../../.secretnote" c.LanguageServerManager.language_servers = { "pyright-extended": { diff --git a/pyprojects/secretnote/pyproject.toml b/pyprojects/secretnote/pyproject.toml index b6239c9..31e110a 100644 --- a/pyprojects/secretnote/pyproject.toml +++ b/pyprojects/secretnote/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ "rich>=13.7.0", "stack_data>=0.6.3", "tqdm>=4.66.1", + "dataset>=1.6.2" ] description = "Notebook suite for SecretFlow" dynamic = ["version"] diff --git a/pyprojects/secretnote/src/secretnote/server/app.py b/pyprojects/secretnote/src/secretnote/server/app.py index 6dbc4b2..b143fe4 100644 --- a/pyprojects/secretnote/src/secretnote/server/app.py +++ b/pyprojects/secretnote/src/secretnote/server/app.py @@ -1,12 +1,12 @@ -import sys # noqa: I001 +import sys from jupyter_server.extension.application import ExtensionApp, ExtensionAppJinjaMixin from secretnote._resources import require from . import JUPYTER_SERVER_EXTENSION_MODULE -from .handlers import SinglePageApplicationHandler -from .node.handler import nodes_handlers +from .services.nodes.handlers import nodes_handlers +from .services.pages.handlers import pages_handlers class SecretNoteApp(ExtensionAppJinjaMixin, ExtensionApp): @@ -30,16 +30,7 @@ def template_paths(self): def initialize_handlers(self): routes = [ *nodes_handlers, - ( - r"/secretnote/preview(.*)", - SinglePageApplicationHandler, - {"path": self.static_paths}, - ), - ( - r"/secretnote(.*)", - SinglePageApplicationHandler, - {"path": self.static_paths}, - ), + *pages_handlers, ] self.handlers.extend(routes) diff --git a/pyprojects/secretnote/src/secretnote/server/db.py b/pyprojects/secretnote/src/secretnote/server/db.py deleted file mode 100644 index 2cea8c9..0000000 --- a/pyprojects/secretnote/src/secretnote/server/db.py +++ /dev/null @@ -1,114 +0,0 @@ -import sqlite3 - -queries = { - "SELECT": "SELECT %s FROM %s WHERE %s", - "SELECT_ALL": "SELECT %s FROM %s", - "INSERT": "INSERT INTO %s VALUES(%s)", - "UPDATE": "UPDATE %s SET %s WHERE %s", - "DELETE": "DELETE FROM %s where %s", - "DELETE_ALL": "DELETE FROM %s", - "CREATE_TABLE": "CREATE TABLE IF NOT EXISTS %s(%s)", - "DROP_TABLE": "DROP TABLE %s", -} - - -class DatabaseObject(object): - def __init__(self, data_file): - self.db = sqlite3.connect(data_file, check_same_thread=False) - self.data_file = data_file - - def free(self, cursor): - cursor.close() - - def write(self, query, values=None): - cursor = self.db.cursor() - if values is not None: - cursor.execute(query, list(values)) - else: - cursor.execute(query) - self.db.commit() - return cursor - - def read(self, query, values=None): - cursor = self.db.cursor() - if values is not None: - cursor.execute(query, list(values)) - else: - cursor.execute(query) - return cursor - - def select(self, tables, *args, **kwargs): - vals = ",".join([l for l in args]) # noqa: E741 - locs = ",".join(tables) - conds = " and ".join(["%s=?" % k for k in kwargs]) - subs = [kwargs[k] for k in kwargs] - query = queries["SELECT"] % (vals, locs, conds) - return self.read(query, subs) - - def select_all(self, tables, *args): - vals = ",".join([l for l in args]) # noqa: E741 - locs = ",".join(tables) - query = queries["SELECT_ALL"] % (vals, locs) - return self.read(query) - - def insert(self, table_name, *args): - values = ",".join(["?" for l in args]) # noqa: E741 - query = queries["INSERT"] % (table_name, values) - return self.write(query, args) - - def update(self, table_name, set_args, **kwargs): - updates = ",".join(["%s=?" % k for k in set_args]) - conds = " and ".join(["%s=?" % k for k in kwargs]) - vals = [set_args[k] for k in set_args] - subs = [kwargs[k] for k in kwargs] - query = queries["UPDATE"] % (table_name, updates, conds) - return self.write(query, vals + subs) - - def delete(self, table_name, **kwargs): - conds = " and ".join(["%s=?" % k for k in kwargs]) - subs = [kwargs[k] for k in kwargs] - query = queries["DELETE"] % (table_name, conds) - return self.write(query, subs) - - def delete_all(self, table_name): - query = queries["DELETE_ALL"] % table_name - return self.write(query) - - def create_table(self, table_name, values): - query = queries["CREATE_TABLE"] % (table_name, ",".join(values)) - self.free(self.write(query)) - - def drop_table(self, table_name): - query = queries["DROP_TABLE"] % table_name - self.free(self.write(query)) - - def disconnect(self): - self.db.close() - - -class Table(DatabaseObject): - def __init__(self, data_file, table_name, values): - super(Table, self).__init__(data_file) - self.create_table(table_name, values) - self.table_name = table_name - - def select(self, *args, **kwargs): - return super(Table, self).select([self.table_name], *args, **kwargs) - - def select_all(self, *args): - return super(Table, self).select_all([self.table_name], *args) - - def insert(self, *args): - return super(Table, self).insert(self.table_name, *args) - - def update(self, set_args, **kwargs): - return super(Table, self).update(self.table_name, set_args, **kwargs) - - def delete(self, **kwargs): - return super(Table, self).delete(self.table_name, **kwargs) - - def delete_all(self): - return super(Table, self).delete_all(self.table_name) - - def drop(self): - return super(Table, self).drop_table(self.table_name) diff --git a/pyprojects/secretnote/src/secretnote/server/node/handler.py b/pyprojects/secretnote/src/secretnote/server/node/handler.py deleted file mode 100644 index becbc48..0000000 --- a/pyprojects/secretnote/src/secretnote/server/node/handler.py +++ /dev/null @@ -1,121 +0,0 @@ -import json -from typing import List, Tuple, Type - -from jupyter_server.base.handlers import APIHandler, JupyterHandler -from tornado import web - -from secretnote.server.node.nodemanager import node_table - -try: - from jupyter_client.jsonutil import json_default -except ImportError: - from jupyter_client.jsonutil import date_default as json_default - - -class NodeRootHandler(APIHandler): - @web.authenticated - def get(self): - data = node_table.select_all("*") - result = [] - # print(data) - for i in range(len(data)): - result.append( - { - "id": data[i][0], - "name": data[i][1], - "address": data[i][2], - } - ) - self.finish(json.dumps(result, default=json_default)) - - @web.authenticated - async def post(self): - model = self.get_json_body() - if model is None: - raise web.HTTPError(400, "No JSON data provided") - - node_id = model.get("id", None) - node_name = model.get("name", None) - node_address = model.get("address", None) - - if node_name is not None: - node = node_table.select("*", name=node_name) - if len(node) != 0: - raise web.HTTPError(400, "node name is already in use") - else: - raise web.HTTPError(400, "node name is required") - - if node_address is not None: - node = node_table.select("*", address=node_address) - if len(node) != 0: - raise web.HTTPError(400, "node address is already in use") - else: - raise web.HTTPError(400, "node address is required") - - node_table.insert(node_id, node_name, node_address) - self.finish(json.dumps(model, default=json_default)) - - -class NodeHandler(APIHandler): - @web.authenticated - async def get(self, node_id): - data = node_table.select("*", id=node_id) - result = [] - for i in range(len(data)): - result.append( - { - "id": data[i][0], - "name": data[i][1], - "address": data[i][2], - } - ) - self.finish(json.dumps(result, default=json_default)) - - @web.authenticated - async def patch(self, node_id): - model = self.get_json_body() - if model is None: - raise web.HTTPError(400, "No JSON data provided") - - node_name = model.get("name", None) - node_address = model.get("address", None) - - if node_name is not None: - node = node_table.select("*", name=node_name) - if len(node) != 0: - raise web.HTTPError(400, "node name is already in use") - else: - node_table.update( - { - "name": node_name, - }, - id=node_id, - ) - - if node_address is not None: - node = node_table.select("*", address=node_address) - if len(node) != 0: - raise web.HTTPError(400, "node address is already in use") - else: - node_table.update( - { - "address": node_address, - }, - id=node_id, - ) - - self.finish(json.dumps(model, default=json_default)) - - @web.authenticated - async def delete(self, node_id): - node_table.delete(id=node_id) - self.set_status(204) - self.finish() - - -_node_id_regex = r"(?P\w+-\w+-\w+-\w+-\w+)" - -nodes_handlers: List[Tuple[str, Type[JupyterHandler]]] = [ - (rf"/api/nodes/{_node_id_regex}", NodeHandler), - (r"/api/nodes", NodeRootHandler), -] diff --git a/pyprojects/secretnote/src/secretnote/server/node/nodemanager.py b/pyprojects/secretnote/src/secretnote/server/node/nodemanager.py deleted file mode 100644 index 3ae6cd2..0000000 --- a/pyprojects/secretnote/src/secretnote/server/node/nodemanager.py +++ /dev/null @@ -1,45 +0,0 @@ -import os - -from jupyter_core import paths - -from secretnote.server.db import Table - - -class Node(Table): - def __init__(self, data_file): - super(Node, self).__init__( - data_file, "node", ["id TEXT", "name TEXT", "address TEXT"] - ) - - def select_all(self, *args): - cursor = super(Node, self).select_all(*args) - results = cursor.fetchall() - cursor.close() - return results - - def select(self, *args, **kwargs): - cursor = super(Node, self).select(*args, **kwargs) - results = cursor.fetchall() - cursor.close() - return results - - def insert(self, *args): - self.free(super(Node, self).insert(*args)) - - def update(self, set_args, **kwargs): - self.free(super(Node, self).update(set_args, **kwargs)) - - def delete(self, **kwargs): - self.free(super(Node, self).delete(**kwargs)) - - def delete_all(self): - self.free(super(Node, self).delete_all()) - - def drop(self): - self.free(super(Node, self).drop()) - - -db_dir = paths.jupyter_config_dir() -if not os.path.exists(db_dir): - os.makedirs(db_dir) -node_table = Node(db_dir + "/secretnote.db") diff --git a/pyprojects/secretnote/src/secretnote/server/services/nodes/__init__.py b/pyprojects/secretnote/src/secretnote/server/services/nodes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyprojects/secretnote/src/secretnote/server/services/nodes/handlers.py b/pyprojects/secretnote/src/secretnote/server/services/nodes/handlers.py new file mode 100644 index 0000000..7406b48 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/server/services/nodes/handlers.py @@ -0,0 +1,88 @@ +import json +from typing import List, Tuple, Type + +from jupyter_client.jsonutil import json_default +from jupyter_server.base.handlers import APIHandler, JupyterHandler +from tornado import web + +from .nodemanager import node_manager + + +class NodeRootHandler(APIHandler): + @web.authenticated + def get(self): + nodes = node_manager.get_nodes() + self.finish(json.dumps(nodes, default=json_default)) + + @web.authenticated + async def post(self): + model = self.get_json_body() + + if model is None: + raise web.HTTPError(400, "no request body provided.") + + node_name = model.get("name", None) + node_address = model.get("address", None) + + if node_name is not None: + node = node_manager.get_node(name=node_name) + if node is not None: + raise web.HTTPError(400, "node name is already existed.") + else: + raise web.HTTPError(400, "node name is required.") + + if node_address is not None: + node = node_manager.get_node(address=node_address) + if node is not None: + raise web.HTTPError(400, "node address is already existed.") + else: + raise web.HTTPError(400, "node address is required.") + + node_id = node_manager.add_node(model) + self.finish(json.dumps({**model, "id": node_id}, default=json_default)) + + +class NodeHandler(APIHandler): + @web.authenticated + async def get(self, node_id): + node = node_manager.get_node(id=node_id) + self.finish(json.dumps(node, default=json_default)) + + @web.authenticated + async def patch(self, node_id): + model = self.get_json_body() + if model is None: + raise web.HTTPError(400, "no request body provided.") + + node_name = model.get("name", None) + node_address = model.get("address", None) + + if node_name is not None: + node = node_manager.get_node(name=node_name) + if node is not None: + raise web.HTTPError(400, "node name is already existed.") + else: + node_manager.update_node(node_id, {"name": node_name}) + + if node_address is not None: + node = node_manager.get_node(address=node_address) + if node is not None: + raise web.HTTPError(400, "node address is already existed.") + else: + node_manager.update_node(node_id, {"address": node_address}) + + self.finish(json.dumps(model, default=json_default)) + + @web.authenticated + async def delete(self, node_id): + node_manager.remove_node(node_id) + self.set_status(204) + self.finish() + + +_node_id_regex = r"(?P\d+)" + +nodes_handlers: List[Tuple[str, Type[JupyterHandler]]] = [ + (rf"/api/nodes/{_node_id_regex}", NodeHandler), + (r"/api/nodes", NodeRootHandler), +] diff --git a/pyprojects/secretnote/src/secretnote/server/services/nodes/nodemanager.py b/pyprojects/secretnote/src/secretnote/server/services/nodes/nodemanager.py new file mode 100644 index 0000000..6e8b5a5 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/server/services/nodes/nodemanager.py @@ -0,0 +1,30 @@ +from dataset import connect + +from secretnote.utils.path import get_db_path + + +class NodeManager: + def __init__(self): + db = connect(get_db_path()) + self.table = db["node"] + + def add_node(self, node): + return self.table.insert(node) + + def remove_node(self, node_id): + self.table.delete(id=node_id) + + def get_node(self, **kwargs): + return self.table.find_one(**kwargs) + + def get_nodes(self): + result = [] + for node in self.table.all(): + result.append(node) + return result + + def update_node(self, node_id, node): + self.table.update({"id": node_id, **node}, ["id"]) + + +node_manager = NodeManager() diff --git a/pyprojects/secretnote/src/secretnote/server/services/pages/__init__.py b/pyprojects/secretnote/src/secretnote/server/services/pages/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyprojects/secretnote/src/secretnote/server/handlers.py b/pyprojects/secretnote/src/secretnote/server/services/pages/handlers.py similarity index 55% rename from pyprojects/secretnote/src/secretnote/server/handlers.py rename to pyprojects/secretnote/src/secretnote/server/services/pages/handlers.py index c36c4ef..b7c8099 100644 --- a/pyprojects/secretnote/src/secretnote/server/handlers.py +++ b/pyprojects/secretnote/src/secretnote/server/services/pages/handlers.py @@ -1,11 +1,20 @@ +from typing import Dict, List, Tuple, Type + import tornado -from jupyter_server.base.handlers import FileFindHandler +from jupyter_server.base.handlers import FileFindHandler, JupyterHandler from jupyter_server.extension.handler import ( ExtensionHandlerJinjaMixin, ExtensionHandlerMixin, ) from tornado import web +from secretnote._resources import require + + +def static_paths(): + package = require.package_of("@secretflow/secretnote/index.html") + return [package.path.joinpath("dist")] + class SinglePageApplicationHandler( ExtensionHandlerJinjaMixin, @@ -26,3 +35,19 @@ async def get(self, path: str = "/"): except web.HTTPError: self.clear() self.write(self.render_template("index.html")) + + +single_page_static_path = static_paths() + +pages_handlers: List[Tuple[str, Type[JupyterHandler], Dict]] = [ + ( + r"/secretnote/preview(.*)", + SinglePageApplicationHandler, + {"path": single_page_static_path}, + ), + ( + r"/secretnote(.*)", + SinglePageApplicationHandler, + {"path": single_page_static_path}, + ), +] diff --git a/pyprojects/secretnote/src/secretnote/utils/path.py b/pyprojects/secretnote/src/secretnote/utils/path.py index ae036f4..8982493 100644 --- a/pyprojects/secretnote/src/secretnote/utils/path.py +++ b/pyprojects/secretnote/src/secretnote/utils/path.py @@ -1,7 +1,9 @@ from collections import deque +from os import makedirs, path from pathlib import Path from typing import Deque, Iterable, Tuple +from jupyter_core import paths from rich.tree import Tree @@ -24,3 +26,10 @@ def path_to_tree(root: Path) -> Tree: subtree = parent.add(item.name) queue.append((subtree, item)) return tree + + +def get_db_path(): + jupyter_config_dir = paths.jupyter_config_dir() + if not path.exists(jupyter_config_dir): + makedirs(jupyter_config_dir) + return f"sqlite:///{jupyter_config_dir}/secretnote.db" diff --git a/requirements-dev.lock b/requirements-dev.lock index c1de68c..5a9d247 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -211,7 +211,6 @@ snakeviz==2.2.0 sniffio==1.3.0 soupsieve==2.5 spu==0.6.0b0 -sqlalchemy==2.0.23 sqlparse==0.4.4 stack-data==0.6.3 starlette==0.27.0 @@ -251,6 +250,11 @@ widgetsnbextension==4.0.9 wrapt==1.16.0 yarl==1.9.3 zipp==3.17.0 +dataset==1.6.2 +alembic==1.12.1 +sqlalchemy==1.4.50 +banal==1.0.6 +mako==1.3.0 # The following packages are considered to be unsafe in a requirements file: pip==23.3.1 setuptools==69.0.2 diff --git a/requirements.lock b/requirements.lock index ce97c17..9d17202 100644 --- a/requirements.lock +++ b/requirements.lock @@ -131,7 +131,6 @@ six==1.16.0 smart-open==6.4.0 sniffio==1.3.0 soupsieve==2.5 -sqlalchemy==2.0.23 sqlparse==0.4.4 stack-data==0.6.3 terminado==0.18.0 @@ -153,3 +152,8 @@ widgetsnbextension==4.0.9 wrapt==1.16.0 yarl==1.9.3 zipp==3.17.0 +dataset==1.6.2 +alembic==1.12.1 +sqlalchemy==1.4.50 +banal==1.0.6 +mako==1.3.0