Add database

.gitignore

@@ -30,3 +30,4 @@ lib/
 
 # Hatch version
 _version.py
+*.sqlite

CONTRIBUTING.md

@@ -49,7 +49,7 @@ docker pull quay.io/jupyterhub/repo2docker:main
 Finally, start `jupyterhub` with the config in `debug` mode:
 
 ```bash
-python -m jupyterhub -f jupyterhub_config.py --debug
+python -m jupyterhub -f ui-tests/jupyterhub_config.py --debug
 ```
 
 Open https://localhost:8000 in a web browser.

dev-requirements.txt

@@ -1,5 +1,6 @@
 git+https://github.com/jupyterhub/the-littlest-jupyterhub@1.0.0
 jupyterhub>=4,<5
+alembic>=1.13.0,<1.14
 pytest
 pytest-aiohttp
 pytest-asyncio

pyproject.toml

@@ -7,7 +7,7 @@ dependencies = [
   "aiodocker~=0.19",
   "dockerspawner~=12.1",
   "jupyter_client>=6.1,<8",
-  "httpx"
+  "httpx",
 ]
 dynamic = ["version"]
 license = {file = "LICENSE"}

tljh_repo2docker/__init__.py

@@ -3,7 +3,6 @@
 from jinja2 import BaseLoader, Environment
 from jupyter_client.localinterfaces import public_ips
 from jupyterhub.traitlets import ByteSpecification
-from tljh.configurer import load_config
 from tljh.hooks import hookimpl
 from traitlets import Unicode
 from traitlets.config import Configurable

tljh_repo2docker/alembic/alembic.ini

@@ -0,0 +1,66 @@
+# A generic, single database configuration.
+
+[alembic]
+script_location = {alembic_dir}
+sqlalchemy.url = {db_url}
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to jupyterhub/alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat jupyterhub/alembic/versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

tljh_repo2docker/alembic/env.py

@@ -0,0 +1,65 @@
+import asyncio
+import logging
+from logging.config import fileConfig
+
+import alembic
+from sqlalchemy import engine_from_config, pool
+from sqlalchemy.ext.asyncio import AsyncEngine
+
+# Alembic Config object, which provides access to values within the .ini file
+config = alembic.context.config
+
+# Interpret the config file for logging
+fileConfig(config.config_file_name)
+logger = logging.getLogger("alembic.env")
+
+
+def run_migrations_online() -> None:
+    """
+    Run migrations in 'online' mode
+    """
+    connectable = config.attributes.get("connection", None)
+
+    if connectable is None:
+        connectable = AsyncEngine(
+            engine_from_config(
+                config.get_section(config.config_ini_section),
+                prefix="sqlalchemy.",
+                poolclass=pool.NullPool,
+            )
+        )
+
+    if isinstance(connectable, AsyncEngine):
+        asyncio.run(run_async_migrations(connectable))
+    else:
+        do_run_migrations(connectable)
+
+
+def do_run_migrations(connection):
+    alembic.context.configure(connection=connection, target_metadata=None)
+    with alembic.context.begin_transaction():
+        alembic.context.run_migrations()
+
+
+async def run_async_migrations(connectable):
+    async with connectable.connect() as connection:
+        await connection.run_sync(do_run_migrations)
+    await connectable.dispose()
+
+
+def run_migrations_offline() -> None:
+    """
+    Run migrations in 'offline' mode.
+    """
+    alembic.context.configure(url=config.get_main_option("sqlalchemy.url"))
+
+    with alembic.context.begin_transaction():
+        alembic.context.run_migrations()
+
+
+if alembic.context.is_offline_mode():
+    logger.info("Running migrations offline")
+    run_migrations_offline()
+else:
+    logger.info("Running migrations online")
+    run_migrations_online()

tljh_repo2docker/alembic/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

tljh_repo2docker/alembic/versions/ac1b4e7e52f3_first_migration.py

@@ -0,0 +1,30 @@
+"""First migration
+
+Revision ID: ac1b4e7e52f3
+Revises:
+Create Date: 2024-04-05 16:25:18.246631
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "ac1b4e7e52f3"
+down_revision = None
+branch_labels = None
+depends_on = None
+
+import sqlalchemy as sa  # noqa
+from alembic import op  # noqa
+from jupyterhub.orm import JSONDict  # noqa
+
+
+def upgrade():
+    op.create_table(
+        "images",
+        sa.Column("uid", sa.Unicode(36)),
+        sa.Column("name", sa.Unicode(4096)),
+        sa.Column("metadata", JSONDict, nullable=True),
+    )
+
+
+def downgrade():
+    op.drop_table("images")

tljh_repo2docker/app.py

@@ -3,6 +3,7 @@
 import socket
 import typing as tp
 from pathlib import Path
+from urllib.parse import urlparse
 
 from jinja2 import Environment, PackageLoader
 from jupyterhub.app import DATA_FILES_PATH
@@ -13,6 +14,7 @@
 from traitlets.config.application import Application
 
 from .builder import BuildHandler
+from .dbutil import async_session_context_factory, sync_to_async_url, upgrade_if_needed
 from .environments import EnvironmentsHandler
 from .logs import LogsHandler
 from .servers import ServersHandler
@@ -118,16 +120,35 @@ def _default_log_level(self):
         allow_none=True,
     )
 
+    db_url = Unicode(
+        "sqlite:///tljh_repo2docker.sqlite",
+        help="url for the database.",
+    ).tag(config=True)
+
+    config_file = Unicode(
+        "tljh_repo2docker_config.py",
+        help="""
+        Config file to load.
+
+        If a relative path is provided, it is taken relative to current directory
+        """,
+        config=True,
+    )
+
     aliases = {
         "port": "TljhRepo2Docker.port",
         "ip": "TljhRepo2Docker.ip",
+        "config": "TljhRepo2Docker.config_file",
         "default_memory_limit": "TljhRepo2Docker.default_memory_limit",
         "default_cpu_limit": "TljhRepo2Docker.default_cpu_limit",
         "machine_profiles": "TljhRepo2Docker.machine_profiles",
     }
 
     def init_settings(self) -> tp.Dict:
         """Initialize settings for the service application."""
+
+        self.load_config_file(self.config_file)
+
         static_path = DATA_FILES_PATH + "/static/"
         static_url_prefix = self.service_prefix + "static/"
         env_opt = {"autoescape": True}
@@ -151,6 +172,8 @@ def init_settings(self) -> tp.Dict:
             default_cpu_limit=self.default_cpu_limit,
             machine_profiles=self.machine_profiles,
         )
+        if hasattr(self, "db_context"):
+            settings["db_context"] = self.db_context
         return settings
 
     def init_handlers(self) -> tp.List:
@@ -196,6 +219,25 @@ def init_handlers(self) -> tp.List:
 
         return handlers
 
+    def init_db(self):
+        async_db_url = sync_to_async_url(self.db_url)
+        urlinfo = urlparse(async_db_url)
+        if urlinfo.password:
+            # avoid logging the database password
+            urlinfo = urlinfo._replace(
+                netloc=f"{urlinfo.username}:[redacted]@{urlinfo.hostname}:{urlinfo.port}"
+            )
+            db_log_url = urlinfo.geturl()
+        else:
+            db_log_url = async_db_url
+        self.log.info("Connecting to db: %s", db_log_url)
+        upgrade_if_needed(async_db_url, log=self.log)
+        try:
+            self.db_context = async_session_context_factory(async_db_url)
+        except Exception:
+            self.log.error("Failed to connect to db: %s", db_log_url)
+            self.log.debug("Database error was:", exc_info=True)
+
     def make_app(self) -> web.Application:
         """Create the tornado web application.
         Returns:
@@ -208,6 +250,7 @@ def make_app(self) -> web.Application:
 
     def start(self):
         """Start the server."""
+        self.init_db()
         settings = self.init_settings()
 
         self.app = web.Application(**settings)