From 8c46d73fcc7b18874f3f71bdcaa07d89b39cb168 Mon Sep 17 00:00:00 2001 From: Duc Trung Le Date: Fri, 5 Apr 2024 23:53:49 +0200 Subject: [PATCH 01/20] Add database --- .gitignore | 1 + CONTRIBUTING.md | 2 +- dev-requirements.txt | 1 + pyproject.toml | 2 +- tljh_repo2docker/__init__.py | 1 - tljh_repo2docker/alembic/alembic.ini | 66 ++++++ tljh_repo2docker/alembic/env.py | 65 ++++++ tljh_repo2docker/alembic/script.py.mako | 24 ++ .../versions/ac1b4e7e52f3_first_migration.py | 30 +++ tljh_repo2docker/app.py | 43 ++++ tljh_repo2docker/dbutil.py | 217 ++++++++++++++++++ tljh_repo2docker/tests/utils.py | 9 +- ui-tests/binderhub_config.py | 34 +++ .../jupyterhub_config.py | 8 +- ui-tests/playwright.config.js | 2 +- ui-tests/tljh_repo2docker_config.py | 7 + 16 files changed, 500 insertions(+), 12 deletions(-) create mode 100644 tljh_repo2docker/alembic/alembic.ini create mode 100644 tljh_repo2docker/alembic/env.py create mode 100644 tljh_repo2docker/alembic/script.py.mako create mode 100644 tljh_repo2docker/alembic/versions/ac1b4e7e52f3_first_migration.py create mode 100644 tljh_repo2docker/dbutil.py create mode 100644 ui-tests/binderhub_config.py rename jupyterhub_config.py => ui-tests/jupyterhub_config.py (85%) create mode 100644 ui-tests/tljh_repo2docker_config.py diff --git a/.gitignore b/.gitignore index 0f8645b..41869a8 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,4 @@ lib/ # Hatch version _version.py +*.sqlite \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 459c00f..4c651ad 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,7 +49,7 @@ docker pull quay.io/jupyterhub/repo2docker:main Finally, start `jupyterhub` with the config in `debug` mode: ```bash -python -m jupyterhub -f jupyterhub_config.py --debug +python -m jupyterhub -f ui-tests/jupyterhub_config.py --debug ``` Open https://localhost:8000 in a web browser. diff --git a/dev-requirements.txt b/dev-requirements.txt index 5a9414a..543fee8 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,6 @@ git+https://github.com/jupyterhub/the-littlest-jupyterhub@1.0.0 jupyterhub>=4,<5 +alembic>=1.13.0,<1.14 pytest pytest-aiohttp pytest-asyncio diff --git a/pyproject.toml b/pyproject.toml index 8a3f29f..66e6b45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ dependencies = [ "aiodocker~=0.19", "dockerspawner~=12.1", "jupyter_client>=6.1,<8", - "httpx" + "httpx", ] dynamic = ["version"] license = {file = "LICENSE"} diff --git a/tljh_repo2docker/__init__.py b/tljh_repo2docker/__init__.py index 8d96e4d..9a5b106 100644 --- a/tljh_repo2docker/__init__.py +++ b/tljh_repo2docker/__init__.py @@ -3,7 +3,6 @@ from jinja2 import BaseLoader, Environment from jupyter_client.localinterfaces import public_ips from jupyterhub.traitlets import ByteSpecification -from tljh.configurer import load_config from tljh.hooks import hookimpl from traitlets import Unicode from traitlets.config import Configurable diff --git a/tljh_repo2docker/alembic/alembic.ini b/tljh_repo2docker/alembic/alembic.ini new file mode 100644 index 0000000..a7354c4 --- /dev/null +++ b/tljh_repo2docker/alembic/alembic.ini @@ -0,0 +1,66 @@ +# A generic, single database configuration. + +[alembic] +script_location = {alembic_dir} +sqlalchemy.url = {db_url} + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to jupyterhub/alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat jupyterhub/alembic/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/tljh_repo2docker/alembic/env.py b/tljh_repo2docker/alembic/env.py new file mode 100644 index 0000000..6188e7e --- /dev/null +++ b/tljh_repo2docker/alembic/env.py @@ -0,0 +1,65 @@ +import asyncio +import logging +from logging.config import fileConfig + +import alembic +from sqlalchemy import engine_from_config, pool +from sqlalchemy.ext.asyncio import AsyncEngine + +# Alembic Config object, which provides access to values within the .ini file +config = alembic.context.config + +# Interpret the config file for logging +fileConfig(config.config_file_name) +logger = logging.getLogger("alembic.env") + + +def run_migrations_online() -> None: + """ + Run migrations in 'online' mode + """ + connectable = config.attributes.get("connection", None) + + if connectable is None: + connectable = AsyncEngine( + engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + ) + + if isinstance(connectable, AsyncEngine): + asyncio.run(run_async_migrations(connectable)) + else: + do_run_migrations(connectable) + + +def do_run_migrations(connection): + alembic.context.configure(connection=connection, target_metadata=None) + with alembic.context.begin_transaction(): + alembic.context.run_migrations() + + +async def run_async_migrations(connectable): + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + await connectable.dispose() + + +def run_migrations_offline() -> None: + """ + Run migrations in 'offline' mode. + """ + alembic.context.configure(url=config.get_main_option("sqlalchemy.url")) + + with alembic.context.begin_transaction(): + alembic.context.run_migrations() + + +if alembic.context.is_offline_mode(): + logger.info("Running migrations offline") + run_migrations_offline() +else: + logger.info("Running migrations online") + run_migrations_online() diff --git a/tljh_repo2docker/alembic/script.py.mako b/tljh_repo2docker/alembic/script.py.mako new file mode 100644 index 0000000..43c0940 --- /dev/null +++ b/tljh_repo2docker/alembic/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/tljh_repo2docker/alembic/versions/ac1b4e7e52f3_first_migration.py b/tljh_repo2docker/alembic/versions/ac1b4e7e52f3_first_migration.py new file mode 100644 index 0000000..fe7bee3 --- /dev/null +++ b/tljh_repo2docker/alembic/versions/ac1b4e7e52f3_first_migration.py @@ -0,0 +1,30 @@ +"""First migration + +Revision ID: ac1b4e7e52f3 +Revises: +Create Date: 2024-04-05 16:25:18.246631 + +""" + +# revision identifiers, used by Alembic. +revision = "ac1b4e7e52f3" +down_revision = None +branch_labels = None +depends_on = None + +import sqlalchemy as sa # noqa +from alembic import op # noqa +from jupyterhub.orm import JSONDict # noqa + + +def upgrade(): + op.create_table( + "images", + sa.Column("uid", sa.Unicode(36)), + sa.Column("name", sa.Unicode(4096)), + sa.Column("metadata", JSONDict, nullable=True), + ) + + +def downgrade(): + op.drop_table("images") diff --git a/tljh_repo2docker/app.py b/tljh_repo2docker/app.py index f48fd76..f73f877 100644 --- a/tljh_repo2docker/app.py +++ b/tljh_repo2docker/app.py @@ -3,6 +3,7 @@ import socket import typing as tp from pathlib import Path +from urllib.parse import urlparse from jinja2 import Environment, PackageLoader from jupyterhub.app import DATA_FILES_PATH @@ -13,6 +14,7 @@ from traitlets.config.application import Application from .builder import BuildHandler +from .dbutil import async_session_context_factory, sync_to_async_url, upgrade_if_needed from .environments import EnvironmentsHandler from .logs import LogsHandler from .servers import ServersHandler @@ -118,9 +120,25 @@ def _default_log_level(self): allow_none=True, ) + db_url = Unicode( + "sqlite:///tljh_repo2docker.sqlite", + help="url for the database.", + ).tag(config=True) + + config_file = Unicode( + "tljh_repo2docker_config.py", + help=""" + Config file to load. + + If a relative path is provided, it is taken relative to current directory + """, + config=True, + ) + aliases = { "port": "TljhRepo2Docker.port", "ip": "TljhRepo2Docker.ip", + "config": "TljhRepo2Docker.config_file", "default_memory_limit": "TljhRepo2Docker.default_memory_limit", "default_cpu_limit": "TljhRepo2Docker.default_cpu_limit", "machine_profiles": "TljhRepo2Docker.machine_profiles", @@ -128,6 +146,9 @@ def _default_log_level(self): def init_settings(self) -> tp.Dict: """Initialize settings for the service application.""" + + self.load_config_file(self.config_file) + static_path = DATA_FILES_PATH + "/static/" static_url_prefix = self.service_prefix + "static/" env_opt = {"autoescape": True} @@ -151,6 +172,8 @@ def init_settings(self) -> tp.Dict: default_cpu_limit=self.default_cpu_limit, machine_profiles=self.machine_profiles, ) + if hasattr(self, "db_context"): + settings["db_context"] = self.db_context return settings def init_handlers(self) -> tp.List: @@ -196,6 +219,25 @@ def init_handlers(self) -> tp.List: return handlers + def init_db(self): + async_db_url = sync_to_async_url(self.db_url) + urlinfo = urlparse(async_db_url) + if urlinfo.password: + # avoid logging the database password + urlinfo = urlinfo._replace( + netloc=f"{urlinfo.username}:[redacted]@{urlinfo.hostname}:{urlinfo.port}" + ) + db_log_url = urlinfo.geturl() + else: + db_log_url = async_db_url + self.log.info("Connecting to db: %s", db_log_url) + upgrade_if_needed(async_db_url, log=self.log) + try: + self.db_context = async_session_context_factory(async_db_url) + except Exception: + self.log.error("Failed to connect to db: %s", db_log_url) + self.log.debug("Database error was:", exc_info=True) + def make_app(self) -> web.Application: """Create the tornado web application. Returns: @@ -208,6 +250,7 @@ def make_app(self) -> web.Application: def start(self): """Start the server.""" + self.init_db() settings = self.init_settings() self.app = web.Application(**settings) diff --git a/tljh_repo2docker/dbutil.py b/tljh_repo2docker/dbutil.py new file mode 100644 index 0000000..73683d6 --- /dev/null +++ b/tljh_repo2docker/dbutil.py @@ -0,0 +1,217 @@ +import os +import shutil +import sys +from contextlib import asynccontextmanager, contextmanager +from datetime import datetime +from pathlib import Path +from subprocess import check_call +from tempfile import TemporaryDirectory +from typing import AsyncGenerator, List +from urllib.parse import urlparse + +import alembic +import alembic.config +from alembic.script import ScriptDirectory +from sqlalchemy import create_engine, inspect, text +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +HERE = Path(__file__).parent.resolve() +ALEMBIC_DIR = HERE / "alembic" +ALEMBIC_INI_TEMPLATE_PATH = ALEMBIC_DIR / "alembic.ini" + + +def write_alembic_ini(alembic_ini: Path, db_url="sqlite:///tljh_repo2docker.sqlite"): + """Write a complete alembic.ini from our template. + + Parameters + ---------- + alembic_ini : str + path to the alembic.ini file that should be written. + db_url : str + The SQLAlchemy database url, e.g. `sqlite:///tljh_repo2docker.sqlite`. + """ + with open(ALEMBIC_INI_TEMPLATE_PATH) as f: + alembic_ini_tpl = f.read() + + with open(alembic_ini, "w") as f: + f.write( + alembic_ini_tpl.format( + alembic_dir=ALEMBIC_DIR, + db_url=str(db_url).replace("%", "%%"), + ) + ) + + +@contextmanager +def _temp_alembic_ini(db_url): + """Context manager for temporary JupyterHub alembic directory + + Temporarily write an alembic.ini file for use with alembic migration scripts. + + Context manager yields alembic.ini path. + + Parameters + ---------- + db_url : str + The SQLAlchemy database url. + + Returns + ------- + alembic_ini: str + The path to the temporary alembic.ini that we have created. + This file will be cleaned up on exit from the context manager. + """ + with TemporaryDirectory() as td: + alembic_ini = Path(td) / "alembic.ini" + write_alembic_ini(alembic_ini, db_url) + yield alembic_ini + + +def upgrade(db_url, revision="head"): + """Upgrade the given database to revision. + + db_url: str + The SQLAlchemy database url. + + revision: str [default: head] + The alembic revision to upgrade to. + """ + with _temp_alembic_ini(db_url) as alembic_ini: + check_call(["alembic", "-c", alembic_ini, "upgrade", revision]) + + +def backup_db_file(db_file, log=None): + """Backup a database file if it exists""" + timestamp = datetime.now().strftime(".%Y-%m-%d-%H%M%S") + backup_db_file = db_file + timestamp + for i in range(1, 10): + if not os.path.exists(backup_db_file): + break + backup_db_file = f"{db_file}.{timestamp}.{i}" + # + if os.path.exists(backup_db_file): + raise OSError("backup db file already exists: %s" % backup_db_file) + if log: + log.info("Backing up %s => %s", db_file, backup_db_file) + shutil.copy(db_file, backup_db_file) + + +def _alembic(db_url: str, alembic_arg: List[str]): + """Run an alembic command with a temporary alembic.ini""" + + with _temp_alembic_ini(db_url) as alembic_ini: + check_call(["alembic", "-c", alembic_ini] + alembic_arg) + + +def check_db_revision(engine): + """Check the database revision""" + # Check database schema version + current_table_names = set(inspect(engine).get_table_names()) + + # alembic needs the password if it's in the URL + engine_url = engine.url.render_as_string(hide_password=False) + + if "alembic_version" not in current_table_names: + return True + + with _temp_alembic_ini(engine_url) as ini: + cfg = alembic.config.Config(ini) + scripts = ScriptDirectory.from_config(cfg) + head = scripts.get_heads()[0] + + # check database schema version + # it should always be defined at this point + with engine.begin() as connection: + alembic_revision = connection.execute( + text("SELECT version_num FROM alembic_version") + ).first()[0] + if alembic_revision == head: + return False + else: + raise Exception( + f"Found database schema version {alembic_revision} != {head}. " + "Backup your database and run `tljh_repo2docker-upgrade-db`" + " to upgrade to the latest schema." + ) + + +def upgrade_if_needed(db_url, log=None): + """Upgrade a database if needed + + If the database is sqlite, a backup file will be created with a timestamp. + Other database systems should perform their own backups prior to calling this. + """ + # run check-db-revision first + engine = create_engine(async_to_sync_url(db_url)) + need_upgrade = check_db_revision(engine=engine) + if not need_upgrade: + if log: + log.info("Database schema is up-to-date") + return + + urlinfo = urlparse(db_url) + if urlinfo.password: + # avoid logging the database password + urlinfo = urlinfo._replace( + netloc=f"{urlinfo.username}:[redacted]@{urlinfo.hostname}:{urlinfo.port}" + ) + db_log_url = urlinfo.geturl() + else: + db_log_url = db_url + if log: + log.info("Upgrading %s", db_log_url) + + upgrade(db_url) + + +def sync_to_async_url(db_url: str) -> str: + """Convert a sync database URL to async one""" + if db_url.startswith("sqlite:"): + return db_url.replace("sqlite:", "sqlite+aiosqlite:") + if db_url.startswith("postgresql:"): + return db_url.replace("postgresql:", "postgresql+asyncpg:") + if db_url.startswith("mysql:"): + return db_url.replace("mysql:", "mysql+aiomysql:") + return db_url + + +def async_to_sync_url(db_url: str) -> str: + """Convert a async database URL to sync one""" + if db_url.startswith("sqlite+aiosqlite:"): + return db_url.replace("sqlite+aiosqlite:", "sqlite:") + if db_url.startswith("postgresql+asyncpg:"): + return db_url.replace("postgresql+asyncpg:", "postgresql:") + if db_url.startswith("mysql+aiomysql:"): + return db_url.replace("mysql+aiomysql:", "mysql:") + return db_url + + +def async_session_context_factory(async_db_url: str): + async_engine = create_async_engine(async_db_url) + async_session_maker = async_sessionmaker( + async_engine, + class_=AsyncSession, + expire_on_commit=False, + autocommit=False, + autoflush=False, + ) + + async def get_async_session() -> AsyncGenerator[AsyncSession, None]: + async with async_session_maker() as session: + yield session + + async_session_context = asynccontextmanager(get_async_session) + return async_session_context + + +def main(args=None): + if args is None: + db_url = sys.argv[1] + alembic_args = sys.argv[2:] + # dumb option parsing, since we want to pass things through + # to subcommands + _alembic(db_url, alembic_args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tljh_repo2docker/tests/utils.py b/tljh_repo2docker/tests/utils.py index 14d9d3e..3465104 100644 --- a/tljh_repo2docker/tests/utils.py +++ b/tljh_repo2docker/tests/utils.py @@ -2,8 +2,13 @@ import json from aiodocker import Docker, DockerError -from jupyterhub.tests.utils import (async_requests, auth_header, - check_db_locks, public_host, public_url) +from jupyterhub.tests.utils import ( + async_requests, + auth_header, + check_db_locks, + public_host, + public_url, +) from jupyterhub.utils import url_path_join as ujoin from tornado.httputil import url_concat diff --git a/ui-tests/binderhub_config.py b/ui-tests/binderhub_config.py new file mode 100644 index 0000000..af1a6e1 --- /dev/null +++ b/ui-tests/binderhub_config.py @@ -0,0 +1,34 @@ +""" +A development config to test BinderHub locally. + +If you are running BinderHub manually (not via JupyterHub) run +`python -m binderhub -f binderhub_config.py` + +Override the external access URL for JupyterHub by setting the +environment variable JUPYTERHUB_EXTERNAL_URL +Host IP is needed in a few places +""" + +import os + +from binderhub.build_local import LocalRepo2dockerBuild +from binderhub.quota import LaunchQuota + + +c.BinderHub.debug = True +c.BinderHub.use_registry = False +c.BinderHub.builder_required = False + +c.BinderHub.build_class = LocalRepo2dockerBuild +c.BinderHub.push_secret = "" +c.BinderHub.launch_quota_class = LaunchQuota + +c.BinderHub.hub_url_local = "http://localhost:8000" + +# Assert that we're running as a managed JupyterHub service +# (otherwise c.BinderHub.hub_api_token is needed) +assert os.getenv("JUPYTERHUB_API_TOKEN") +c.BinderHub.base_url = os.getenv("JUPYTERHUB_SERVICE_PREFIX") +# JUPYTERHUB_BASE_URL may not include the host +# c.BinderHub.hub_url = os.getenv('JUPYTERHUB_BASE_URL') +c.BinderHub.hub_url = os.getenv("JUPYTERHUB_EXTERNAL_URL") diff --git a/jupyterhub_config.py b/ui-tests/jupyterhub_config.py similarity index 85% rename from jupyterhub_config.py rename to ui-tests/jupyterhub_config.py index 05be037..6e402e4 100644 --- a/jupyterhub_config.py +++ b/ui-tests/jupyterhub_config.py @@ -33,12 +33,8 @@ "127.0.0.1", "--port", "6789", - "--machine_profiles", - '{"label": "Small", "cpu": 2, "memory": 2}', - "--machine_profiles", - '{"label": "Medium", "cpu": 4, "memory": 4}', - "--machine_profiles", - '{"label": "Large", "cpu": 8, "memory": 8}', + "--config", + "tljh_repo2docker_config.py" ], "oauth_no_confirm": True, "oauth_client_allowed_scopes": [ diff --git a/ui-tests/playwright.config.js b/ui-tests/playwright.config.js index e0d50c6..f229aa7 100644 --- a/ui-tests/playwright.config.js +++ b/ui-tests/playwright.config.js @@ -13,7 +13,7 @@ module.exports = { } }, webServer: { - command: 'python -m jupyterhub -f ../jupyterhub_config.py', + command: 'python -m jupyterhub -f ./jupyterhub_config.py', url: 'http://localhost:8000', timeout: 120 * 1000, reuseExistingServer: true diff --git a/ui-tests/tljh_repo2docker_config.py b/ui-tests/tljh_repo2docker_config.py new file mode 100644 index 0000000..d9de0a6 --- /dev/null +++ b/ui-tests/tljh_repo2docker_config.py @@ -0,0 +1,7 @@ +c.TljhRepo2Docker.db_url = "sqlite:///tljh_repo2docker.sqlite" + +c.TljhRepo2Docker.machine_profiles = [ + {"label": "Small", "cpu": 2, "memory": 2}, + {"label": "Medium", "cpu": 4, "memory": 4}, + {"label": "Large", "cpu": 8, "memory": 8}, +] From ba5200ac2a3bbe56de00eea74defceff00c56a2b Mon Sep 17 00:00:00 2001 From: Duc Trung Le Date: Mon, 8 Apr 2024 22:53:55 +0200 Subject: [PATCH 02/20] Update config --- tljh_repo2docker/dbutil.py | 2 +- ui-tests/binderhub_config.py | 6 ++++-- ui-tests/jupyterhub_config.py | 31 +++++++++++++++++++++++++++++-- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/tljh_repo2docker/dbutil.py b/tljh_repo2docker/dbutil.py index 73683d6..8bd06f4 100644 --- a/tljh_repo2docker/dbutil.py +++ b/tljh_repo2docker/dbutil.py @@ -130,7 +130,7 @@ def check_db_revision(engine): else: raise Exception( f"Found database schema version {alembic_revision} != {head}. " - "Backup your database and run `tljh_repo2docker-upgrade-db`" + "Backup your database and run `tljh_repo2docker upgrade-db`" " to upgrade to the latest schema." ) diff --git a/ui-tests/binderhub_config.py b/ui-tests/binderhub_config.py index af1a6e1..42ad90b 100644 --- a/ui-tests/binderhub_config.py +++ b/ui-tests/binderhub_config.py @@ -24,11 +24,13 @@ c.BinderHub.launch_quota_class = LaunchQuota c.BinderHub.hub_url_local = "http://localhost:8000" +# c.BinderHub.enable_api_only_mode = True # Assert that we're running as a managed JupyterHub service # (otherwise c.BinderHub.hub_api_token is needed) assert os.getenv("JUPYTERHUB_API_TOKEN") + c.BinderHub.base_url = os.getenv("JUPYTERHUB_SERVICE_PREFIX") # JUPYTERHUB_BASE_URL may not include the host -# c.BinderHub.hub_url = os.getenv('JUPYTERHUB_BASE_URL') -c.BinderHub.hub_url = os.getenv("JUPYTERHUB_EXTERNAL_URL") +c.BinderHub.hub_url = os.getenv('JUPYTERHUB_BASE_URL') +# c.BinderHub.hub_url = os.getenv("JUPYTERHUB_EXTERNAL_URL") diff --git a/ui-tests/jupyterhub_config.py b/ui-tests/jupyterhub_config.py index 6e402e4..7df9dd6 100644 --- a/ui-tests/jupyterhub_config.py +++ b/ui-tests/jupyterhub_config.py @@ -3,16 +3,21 @@ and overrides some of the default values from the plugin. """ +import os +from pathlib import Path from jupyterhub.auth import DummyAuthenticator from tljh.configurer import apply_config, load_config from tljh_repo2docker import tljh_custom_jupyterhub_config, TLJH_R2D_ADMIN_SCOPE import sys + +HERE = Path(__file__).parent tljh_config = load_config() apply_config(tljh_config, c) tljh_custom_jupyterhub_config(c) +tljh_repo2docker_config = HERE / "tljh_repo2docker_config.py" c.JupyterHub.authenticator_class = DummyAuthenticator @@ -20,8 +25,30 @@ c.JupyterHub.allow_named_servers = True c.JupyterHub.ip = "0.0.0.0" + +binderhub_service_name = "binder" +binderhub_config = HERE / "binderhub_config.py" + + +binderhub_environment = {} +for env_var in ["JUPYTERHUB_EXTERNAL_URL", "GITHUB_ACCESS_TOKEN"]: + if os.getenv(env_var) is not None: + binderhub_environment[env_var] = os.getenv(env_var) + c.JupyterHub.services.extend( [ + { + "name": binderhub_service_name, + "admin": True, + "command": [ + sys.executable, + "-m", + "binderhub", + f"--config={binderhub_config}", + ], + "url": "http://localhost:8585", + "environment": binderhub_environment, + }, { "name": "tljh_repo2docker", "url": "http://127.0.0.1:6789", @@ -34,13 +61,13 @@ "--port", "6789", "--config", - "tljh_repo2docker_config.py" + f"{tljh_repo2docker_config}", ], "oauth_no_confirm": True, "oauth_client_allowed_scopes": [ TLJH_R2D_ADMIN_SCOPE, ], - } + }, ] ) From 9451464bb31ca060973e01836d56efbd995adadd Mon Sep 17 00:00:00 2001 From: Duc Trung LE Date: Tue, 9 Apr 2024 13:59:37 +0200 Subject: [PATCH 03/20] Add repo provider to interface --- pyproject.toml | 1 + src/environments/App.tsx | 4 + src/environments/NewEnvironmentDialog.tsx | 200 +++++++++++++++------- src/environments/main.tsx | 3 +- tljh_repo2docker/app.py | 23 ++- tljh_repo2docker/environments.py | 7 + tljh_repo2docker/templates/images.html | 2 +- ui-tests/tljh_repo2docker_config.py | 2 + 8 files changed, 177 insertions(+), 65 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 66e6b45..2e5d670 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ dependencies = [ "dockerspawner~=12.1", "jupyter_client>=6.1,<8", "httpx", + "sqlalchemy>=2" ] dynamic = ["version"] license = {file = "LICENSE"} diff --git a/src/environments/App.tsx b/src/environments/App.tsx index 6a3779c..5a301ef 100644 --- a/src/environments/App.tsx +++ b/src/environments/App.tsx @@ -16,6 +16,8 @@ export interface IAppProps { default_cpu_limit: string; default_mem_limit: string; machine_profiles: IMachineProfile[]; + use_binderhub: boolean; + repo_providers?: { label: string; value: string }[]; } export default function App(props: IAppProps) { const jhData = useJupyterhub(); @@ -35,6 +37,8 @@ export default function App(props: IAppProps) { default_cpu_limit={props.default_cpu_limit} default_mem_limit={props.default_mem_limit} machine_profiles={props.machine_profiles} + use_binderhub={props.use_binderhub} + repo_providers={props.repo_providers} /> diff --git a/src/environments/NewEnvironmentDialog.tsx b/src/environments/NewEnvironmentDialog.tsx index a36c9e3..5d6b851 100644 --- a/src/environments/NewEnvironmentDialog.tsx +++ b/src/environments/NewEnvironmentDialog.tsx @@ -13,7 +13,14 @@ import { Select, Typography } from '@mui/material'; -import { Fragment, memo, useCallback, useMemo, useState } from 'react'; +import { + Fragment, + memo, + useCallback, + useEffect, + useMemo, + useState +} from 'react'; import { useAxios } from '../common/AxiosContext'; import { SmallTextField } from '../common/SmallTextField'; @@ -28,9 +35,12 @@ export interface INewEnvironmentDialogProps { default_cpu_limit: string; default_mem_limit: string; machine_profiles: IMachineProfile[]; + use_binderhub: boolean; + repo_providers?: { label: string; value: string }[]; } interface IFormValues { + provider?: string; repo?: string; ref?: string; name?: string; @@ -58,6 +68,7 @@ function _NewEnvironmentDialog(props: INewEnvironmentDialogProps) { event?: any, reason?: 'backdropClick' | 'escapeKeyDown' ) => { + console.log(formValues); if (reason && reason === 'backdropClick') { return; } @@ -78,7 +89,49 @@ function _NewEnvironmentDialog(props: INewEnvironmentDialogProps) { }, [formValues]); const [selectedProfile, setSelectedProfile] = useState(0); + const [selectedProvider, setSelectedProvider] = useState(0); + + const onMachineProfileChange = useCallback( + (value?: string | number) => { + if (value !== undefined) { + const index = parseInt(value + ''); + const selected = props.machine_profiles[index]; + if (selected !== undefined) { + updateFormValue('cpu', selected.cpu + ''); + updateFormValue('memory', selected.memory + ''); + setSelectedProfile(index); + } + } + }, + [props.machine_profiles, updateFormValue] + ); + const onRepoProviderChange = useCallback( + (value?: string | number) => { + if (value !== undefined) { + const index = parseInt(value + ''); + const selected = props.repo_providers?.[index]; + if (selected !== undefined) { + updateFormValue('provider', selected.value); + setSelectedProvider(index); + } + } + }, + [props.repo_providers, updateFormValue] + ); + useEffect(() => { + if (props.machine_profiles.length > 0) { + onMachineProfileChange(0); + } + if (props.repo_providers && props.repo_providers.length > 0) { + onRepoProviderChange(0); + } + }, [ + props.machine_profiles, + props.repo_providers, + onMachineProfileChange, + onRepoProviderChange + ]); const MemoryCpuSelector = useMemo(() => { return ( @@ -120,16 +173,7 @@ function _NewEnvironmentDialog(props: INewEnvironmentDialogProps) { value={selectedProfile} label="Machine profile" size="small" - onChange={e => { - const value = e.target.value; - if (value) { - const index = parseInt(value + ''); - const selected = props.machine_profiles[index]; - updateFormValue('cpu', selected.cpu + ''); - updateFormValue('memory', selected.memory + ''); - setSelectedProfile(index); - } - }} + onChange={e => onMachineProfileChange(e.target.value)} > {props.machine_profiles.map((it, idx) => { return ( @@ -141,7 +185,8 @@ function _NewEnvironmentDialog(props: INewEnvironmentDialogProps) { ); - }, [updateFormValue, props.machine_profiles, selectedProfile]); + }, [props.machine_profiles, selectedProfile, onMachineProfileChange]); + return ( @@ -186,6 +231,29 @@ function _NewEnvironmentDialog(props: INewEnvironmentDialogProps) { > Create a new environment + {props.use_binderhub && props.repo_providers && ( + + + Repository provider + + + + )} 0 ? MachineProfileSelector : MemoryCpuSelector} - - - Advanced - - - updateFormValue('buildargs', e.target.value)} - /> - - - Credentials - - - updateFormValue('username', e.target.value)} - /> - + {!props.use_binderhub && ( + + + + Advanced + + + updateFormValue('buildargs', e.target.value)} + /> + + )} + {!props.use_binderhub && ( + + + + Credentials + + + updateFormValue('username', e.target.value)} + /> + + + )} )} diff --git a/src/servers/ServersList.tsx b/src/servers/ServersList.tsx index 086e605..d65c290 100644 --- a/src/servers/ServersList.tsx +++ b/src/servers/ServersList.tsx @@ -65,7 +65,7 @@ const columns: GridColDef[] = [ ); @@ -86,7 +86,9 @@ function _ServerList(props: IServerListProps) { } const allServers = servers.map((it, id) => { const newItem: any = { ...it, id }; - newItem.image = it.user_options.image ?? ''; + newItem.image = + it.user_options?.display_name ?? it.user_options.image ?? ''; + newItem.uid = it.user_options?.uid ?? null; newItem.last_activity = formatTime(newItem.last_activity); return newItem; }); diff --git a/src/servers/types.ts b/src/servers/types.ts index cf51864..c82723e 100644 --- a/src/servers/types.ts +++ b/src/servers/types.ts @@ -3,6 +3,6 @@ export interface IServerData { name: string; url: string; last_activity: string; - user_options: { image?: string }; + user_options: { image?: string; display_name?: string; uid?: string }; active: boolean; } diff --git a/tljh_repo2docker/binderhub_builder.py b/tljh_repo2docker/binderhub_builder.py index 3086d15..24f7cfa 100644 --- a/tljh_repo2docker/binderhub_builder.py +++ b/tljh_repo2docker/binderhub_builder.py @@ -3,7 +3,7 @@ from urllib.parse import quote from uuid import UUID, uuid4 -from aiodocker import Docker, DockerError +from aiodocker import Docker from jupyterhub.utils import url_path_join from tornado import web @@ -44,11 +44,11 @@ async def delete(self): async with db_context() as db: image = await image_db_manager.read(db, uid) if image: - async with Docker() as docker: - try: + try: + async with Docker() as docker: await docker.images.delete(image.name) - except DockerError: - pass + except Exception: + pass deleted = await image_db_manager.delete(db, uid) self.set_header("content-type", "application/json") diff --git a/tljh_repo2docker/database/manager.py b/tljh_repo2docker/database/manager.py index b252e1f..02918d6 100644 --- a/tljh_repo2docker/database/manager.py +++ b/tljh_repo2docker/database/manager.py @@ -1,5 +1,5 @@ import logging -from typing import List, Type, Union +from typing import List, Optional, Type, Union import sqlalchemy as sa from pydantic import UUID4 @@ -105,6 +105,25 @@ async def read_all(self, db: AsyncSession) -> List[DockerImageOutSchema]: resources = (await db.execute(sa.select(self._table))).scalars().all() return [self._schema_out.model_validate(r) for r in resources] + async def read_by_image_name( + self, db: AsyncSession, image: str + ) -> Optional[DockerImageOutSchema]: + """ + Get image by its name. + + Args: + db: An asyncio version of SQLAlchemy session. + + Returns: + The list of resources retrieved. + """ + statement = sa.select(self._table).where(self._table.name == image) + try: + result = await db.execute(statement) + return self._schema_out.model_validate(result.scalars().first()) + except Exception: + return None + async def update( self, db: AsyncSession, obj_in: DockerImageUpdateSchema, optimistic: bool = True ) -> Union[DockerImageOutSchema, None]: diff --git a/tljh_repo2docker/servers.py b/tljh_repo2docker/servers.py index 5411a70..aa9f1e1 100644 --- a/tljh_repo2docker/servers.py +++ b/tljh_repo2docker/servers.py @@ -1,4 +1,5 @@ from inspect import isawaitable +from typing import Dict, List from tornado import web @@ -24,7 +25,22 @@ async def get(self): user_data = await self.fetch_user() - server_data = user_data.all_spawners() + server_data: List[Dict] = user_data.all_spawners() + + db_context, image_db_manager = self.get_db_handlers() + if db_context and image_db_manager: + async with db_context() as db: + for data in server_data: + image_name = data.get("user_options", {}).get("image", None) + if image_name: + db_data = await image_db_manager.read_by_image_name( + db, image_name + ) + if db_data: + data["user_options"]["uid"] = str(db_data.uid) + data["user_options"][ + "display_name" + ] = db_data.image_meta.display_name named_server_limit = 0 result = self.render_template( "servers.html", diff --git a/tljh_repo2docker/tests/binderhub_build/test_logs.py b/tljh_repo2docker/tests/binderhub_build/test_logs.py index 787cbe6..29123dc 100644 --- a/tljh_repo2docker/tests/binderhub_build/test_logs.py +++ b/tljh_repo2docker/tests/binderhub_build/test_logs.py @@ -1,5 +1,3 @@ -import json - import pytest from jupyterhub.tests.utils import async_requests diff --git a/tljh_repo2docker/tests/local_build/test_logs.py b/tljh_repo2docker/tests/local_build/test_logs.py index ce444ca..077e75a 100644 --- a/tljh_repo2docker/tests/local_build/test_logs.py +++ b/tljh_repo2docker/tests/local_build/test_logs.py @@ -1,5 +1,3 @@ -import json - import pytest from jupyterhub.tests.utils import async_requests diff --git a/ui-tests/jupyterhub_config_binderhub.py b/ui-tests/jupyterhub_config_binderhub.py index f95fd74..10b251f 100644 --- a/ui-tests/jupyterhub_config_binderhub.py +++ b/ui-tests/jupyterhub_config_binderhub.py @@ -58,7 +58,7 @@ "-m", "tljh_repo2docker", "--ip", - "127.0.0.1", + "0.0.0.0", "--port", "6789", "--config",