Add encryption of passwords for remote librarians

librarian_server/encryption.py

@@ -0,0 +1,33 @@
+"""
+Functions for encrypting and decrypting data.
+"""
+
+from cryptography.fernet import Fernet
+
+from .settings import server_settings
+
+
+def encrypt_string(input: str) -> str:
+    """
+    Encrypt the given string.
+    """
+    key = server_settings.encryption_key
+
+    if key is None:
+        raise ValueError("No encryption key is set!")
+
+    f = Fernet(key=key)
+    return f.encrypt(input.encode()).decode()
+
+
+def decrypt_string(input: str) -> str:
+    """
+    Decrypt the given string.
+    """
+    key = server_settings.encryption_key
+
+    if key is None:
+        raise ValueError("No encryption key is set!")
+
+    f = Fernet(key=key)
+    return f.decrypt(input.encode()).decode()

librarian_server/orm/librarian.py

@@ -11,6 +11,9 @@
 from hera_librarian.exceptions import LibrarianHTTPError
 
 from .. import database as db
+from ..encryption import decrypt_string, encrypt_string
+from ..logger import log
+from ..settings import server_settings
 
 
 class Librarian(db.Base):
@@ -32,9 +35,7 @@ class Librarian(db.Base):
     port = db.Column(db.Integer, nullable=False)
     "The port of this librarian."
     authenticator = db.Column(db.String(256), nullable=False)
-    "The authenticator so we can connect this librarian."
-    # TODO: THIS IS A MASSIVE HOLE IN SECURITY THAT ABSOLUTELY MUST BE FIXED
-    # URGENT: FIX THIS.
+    "The authenticator so we can connect this librarian. This is encrypted."
 
     last_seen = db.Column(db.DateTime, nullable=False)
     "The last time we connected to and verified this librarian exists."
@@ -43,7 +44,12 @@ class Librarian(db.Base):
 
     @classmethod
     def new_librarian(
-        self, name: str, url: str, port: int, check_connection: bool = True
+        self,
+        name: str,
+        url: str,
+        port: int,
+        authenticator: str,
+        check_connection: bool = True,
     ) -> "Librarian":
         """
         Create a new librarian object.
@@ -56,6 +62,10 @@ def new_librarian(
             The URL of this librarian.
         port : int
             The port of this librarian.
+        authenticator : str
+            The authenticator so we can connect this librarian. This is passed in
+            unencrypted and will be encrypted before being stored. The authenticator
+            is a username and password separated by a colon.
         check_connection : bool
             Whether to check the connection to this librarian before
             returning it (default: True, but turn this off for tests.)
@@ -64,12 +74,18 @@ def new_librarian(
         -------
         Librarian
             The new librarian.
+
+        Raises
+        ------
+        ValueError
+            No encryption key is set!
         """
 
         librarian = Librarian(
             name=name,
             url=url,
             port=port,
+            authenticator=encrypt_string(authenticator),
             last_seen=datetime.utcnow(),
             last_heard=datetime.utcnow(),
         )
@@ -105,9 +121,11 @@ def client(self) -> LibrarianClient:
             The client.
         """
 
+        decrpyted_authenticator = decrypt_string(self.authenticator)
+
         return LibrarianClient(
             host=self.url,
             port=self.port,
-            user=self.authenticator.split(":")[0],
-            password=self.authenticator.split(":")[1],
+            user=decrpyted_authenticator.split(":")[0],
+            password=decrpyted_authenticator.split(":")[1],
         )

librarian_server/settings.py

@@ -58,6 +58,11 @@ class ServerSettings(BaseSettings):
     # Top level name of the server. Should be unique.
     name: str = "librarian_server"
 
+    # Encryption key for the server, for connecting to other librarians.
+    # Don't write this in the config file, it should be set as an environment
+    # variable.
+    encryption_key: Optional[str] = None
+
     # Database settings.
     database_driver: str = "sqlite"
     database_user: Optional[str] = None
@@ -72,11 +77,14 @@ class ServerSettings(BaseSettings):
     displayed_site_name: str = "Untitled Librarian"
     displayed_site_description: str = "No description set."
 
+    # Host and port to bind to.
     host: str = "0.0.0.0"
     port: int
 
+    # Stores that the librarian should add or migrate
     add_stores: list[StoreSettings]
 
+    # Database migration settings
     alembic_config_path: str = "."
     alembic_path: str = "alembic"
 

pyproject.toml

@@ -22,6 +22,7 @@ dependencies = [
     "checksumdir",
     "python-dateutil",
     "argon2-cffi",
+    "cryptography",
 ]
 authors = [
     {name = "HERA Team", email = "hera@lists.berkeley.edu"},

tests/conftest.py

@@ -61,18 +61,12 @@ def test_server(tmp_path_factory):
 
     setup = server_setup(tmp_path_factory, name="test_server")
 
-    env_vars = {
-        "LIBRARIAN_SERVER_NAME": None,
-        "LIBRARIAN_CONFIG_PATH": None,
-        "LIBRARIAN_SERVER_DATABASE_DRIVER": None,
-        "LIBRARIAN_SERVER_DATABASE": None,
-        "LIBRARIAN_SERVER_PORT": None,
-        "LIBRARIAN_SERVER_ADD_STORES": None,
-    }
+    env_vars = {x: None for x in setup.env.keys()}
 
     for env_var in list(env_vars.keys()):
         env_vars[env_var] = os.environ.get(env_var, None)
-        os.environ[env_var] = getattr(setup, env_var)
+        if setup.env[env_var] is not None:
+            os.environ[env_var] = setup.env[env_var]
 
     global DATABASE_PATH
     DATABASE_PATH = str(setup.database)

tests/integration_test/test_sneaker.py

@@ -31,14 +31,10 @@ def test_sneakernet_workflow(
         live_server = test_orm.Librarian.new_librarian(
             name="live_server",
             url="http://localhost",
+            authenticator="admin:password",  # This is the default authenticator.
             port=server.id,
-            check_connection=False,
         )
 
-        live_server.authenticator = "admin:password"
-
-        live_server.client().ping()
-
         session.add(live_server)
         session.commit()
 
@@ -47,12 +43,11 @@ def test_sneakernet_workflow(
         test_server = test_orm.Librarian.new_librarian(
             name="test_server",
             url="http://localhost",
+            authenticator="admin:password",  # This is the default authenticator.
             port=test_server_with_many_files_and_errors[2].id,
             check_connection=False,
         )
 
-        test_server.authenticator = "admin:password"
-
         session.add(test_server)
         session.commit()
 

tests/server.py

@@ -11,6 +11,7 @@
 from pathlib import Path
 from subprocess import run
 
+from cryptography.fernet import Fernet
 from pydantic import BaseModel
 
 
@@ -24,6 +25,7 @@ class Server(BaseModel):
     LIBRARIAN_SERVER_DISPLAYED_SITE_NAME: str
     LIBRARIAN_CONFIG_PATH: str
     LIBRARIAN_SERVER_DATABASE_DRIVER: str
+    LIBRARIAN_SERVER_ENCRYPTION_KEY: str
     LIBRARIAN_SERVER_DATABASE: str
     LIBRARIAN_SERVER_PORT: str
     LIBRARIAN_SERVER_ADD_STORES: str
@@ -39,6 +41,7 @@ def env(self) -> dict[str, str]:
             "LIBRARIAN_SERVER_NAME": self.LIBRARIAN_SERVER_NAME,
             "LIBRARIAN_SERVER_DISPLAYED_SITE_NAME": self.LIBRARIAN_SERVER_DISPLAYED_SITE_NAME,
             "LIBRARIAN_CONFIG_PATH": self.LIBRARIAN_CONFIG_PATH,
+            "LIBRARIAN_SERVER_ENCRYPTION_KEY": self.LIBRARIAN_SERVER_ENCRYPTION_KEY,
             "LIBRARIAN_SERVER_DATABASE_DRIVER": self.LIBRARIAN_SERVER_DATABASE_DRIVER,
             "LIBRARIAN_SERVER_DATABASE": self.LIBRARIAN_SERVER_DATABASE,
             "LIBRARIAN_SERVER_PORT": self.LIBRARIAN_SERVER_PORT,
@@ -199,6 +202,7 @@ def server_setup(tmp_path_factory, name="librarian_server") -> Server:
         database=database,
         LIBRARIAN_SERVER_NAME=name,
         LIBRARIAN_SERVER_DISPLAYED_SITE_NAME=name.replace("_", " ").title(),
+        LIBRARIAN_SERVER_ENCRYPTION_KEY=Fernet.generate_key().decode(),
         LIBRARIAN_CONFIG_PATH=librarian_config_path,
         LIBRARIAN_SERVER_DATABASE_DRIVER="sqlite",
         LIBRARIAN_SERVER_DATABASE=str(database),

tests/server_unit_test/test_admin.py

@@ -246,7 +246,8 @@ def test_manifest_generation_and_extra_opts(
         librarian = test_orm.Librarian.new_librarian(
             "our_closest_friend",
             "http://localhost",
-            80,
+            authenticator="admin:password",  # This is the default authenticator.
+            port=80,
             check_connection=False,
         )
 

tests/server_unit_test/test_clone.py

@@ -308,7 +308,11 @@ def test_incoming_transfer_endpoints(
         session.commit()
 
         librarian = test_orm.Librarian.new_librarian(
-            name="test2", url="http://localhost", port=5000, check_connection=False
+            name="test2",
+            url="http://localhost",
+            port=5000,
+            check_connection=False,
+            authenticator="admin:password",
         )
         librarian.authenticator = "does_not_authenticate"
         session.add(librarian)

tests/server_unit_test/test_encryption.py

@@ -0,0 +1,36 @@
+"""
+Tests for encryption technology.
+"""
+
+import os
+
+from ..server import server_setup
+
+
+def test_encrypt_decrypt_cycle(tmp_path_factory):
+    setup = server_setup(tmp_path_factory, name="test_server")
+
+    env_vars = {x: None for x in setup.env.keys()}
+
+    for env_var in list(env_vars.keys()):
+        env_vars[env_var] = os.environ.get(env_var, None)
+        if setup.env[env_var] is not None:
+            os.environ[env_var] = setup.env[env_var]
+
+    from librarian_server.encryption import decrypt_string, encrypt_string
+
+    input = "hello:world"
+
+    encrypted = encrypt_string(input)
+
+    assert encrypted != input
+
+    decrypted = decrypt_string(encrypted)
+
+    assert decrypted == input
+
+    for env_var in list(env_vars.keys()):
+        if env_vars[env_var] is None:
+            del os.environ[env_var]
+        else:
+            os.environ[env_var] = env_vars[env_var]