Automatically migrate unencrypted databases on request (#121)

* Automatically migrate unencrypted databases on request

* Format

* Clippy

* Improve error handling without strings

* Format

* Clippy lint ignore

* Skip encrypt migration test on Windows

* Update changelog

* Fail fast false

* Try vcpkg

* Try getting libsodium from vcpkg

* lib dir?

* Lib dir with pwsh

* Set one var and correct path

* Check dir

* More listing

* It's in release

* Finish fixing build

.github/workflows/release.yml

@@ -11,6 +11,7 @@ jobs:
     name: Release Build
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         os: [
           ubuntu-latest,
@@ -23,12 +24,28 @@ jobs:
         ]
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Toolchain
         uses: actions-rs/toolchain@v1
         with:
           toolchain: ${{ matrix.toolchain }}
 
+      - name: Install vcpkg Packages
+        if: matrix.os == 'windows-latest'
+        uses: johnwason/vcpkg-action@v6
+        id: vcpkg
+        with:
+          pkgs: libsodium
+          triplet: x64-windows-release
+          token: ${{ github.token }}
+          github-binarycache: true
+
+      - name: Make Release Windows
+        if: matrix.os == 'windows-latest'
+        run: |-
+          $env:SODIUM_LIB_DIR="$(pwd)\vcpkg\packages\libsodium_x64-windows-release\lib"
+          make release
+
       - name: Make Release
         run: make release

.github/workflows/static.yml

@@ -11,6 +11,7 @@ jobs:
     name: Static Analysis
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         os: [
           ubuntu-latest,
@@ -21,7 +22,7 @@ jobs:
         ]
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Toolchain
         uses: actions-rs/toolchain@v1

.github/workflows/test.yml

@@ -11,6 +11,7 @@ jobs:
     name: Test
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         os: [
           ubuntu-latest,
@@ -23,12 +24,29 @@ jobs:
         ]
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Toolchain
         uses: actions-rs/toolchain@v1
         with:
           toolchain: ${{ matrix.toolchain }}
 
+      - name: Install vcpkg Packages
+        if: matrix.os == 'windows-latest'
+        uses: johnwason/vcpkg-action@v6
+        id: vcpkg
+        with:
+          pkgs: libsodium
+          triplet: x64-windows-release
+          token: ${{ github.token }}
+          github-binarycache: true
+
+      - name: Make Test Windows
+        if: matrix.os == 'windows-latest'
+        run: |-
+          $env:SODIUM_LIB_DIR="$(pwd)\vcpkg\packages\libsodium_x64-windows-release\lib"
+          make test
+
       - name: Make Test
+        if: matrix.os != 'windows-latest'
         run: make test

crates/lair_keystore/CHANGELOG.md

@@ -1 +1,9 @@
+## 0.4.1
+
+- Add a way to migrate unencrypted databases to encrypted by providing an environment variable `LAIR_MIGRATE_UNENCRYPTED="true"`, Lair will detect databases which can't be opened and attempt migration. #121
+
+# 0.4.0
+
+- pin serde and rmp-serde #119
+
 ## 0.0.2

crates/lair_keystore/src/lib.rs

@@ -54,6 +54,8 @@ include!(concat!(env!("OUT_DIR"), "/ver.rs"));
 
 /// Re-exported dependencies.
 pub mod dependencies {
+    // Not sure why Clippy picks this up as unused, it's exported to be used elsewhere
+    #[allow(unused_imports)]
     pub use hc_seed_bundle::dependencies::*;
     pub use lair_keystore_api;
     pub use lair_keystore_api::dependencies::*;
@@ -75,6 +77,3 @@ pub mod store_sqlite;
 
 #[doc(inline)]
 pub use store_sqlite::create_sql_pool_factory;
-
-#[cfg(test)]
-mod server_test;

crates/lair_keystore/src/store_sqlite.rs

@@ -111,20 +111,18 @@ impl SqlCon {
 
 /// extension trait for execute that we don't care about results
 trait ExecExt {
-    fn execute_optional<P>(&self, sql: &str, params: P) -> LairResult<()>
+    fn execute_optional<P>(&self, sql: &str, params: P) -> rusqlite::Result<()>
     where
         P: rusqlite::Params;
 }
 
 impl ExecExt for rusqlite::Connection {
-    fn execute_optional<P>(&self, sql: &str, params: P) -> LairResult<()>
+    fn execute_optional<P>(&self, sql: &str, params: P) -> rusqlite::Result<()>
     where
         P: rusqlite::Params,
     {
         use rusqlite::OptionalExtension;
-        self.query_row(sql, params, |_| Ok(()))
-            .optional()
-            .map_err(one_err::OneErr::new)?;
+        self.query_row(sql, params, |_| Ok(())).optional()?;
         Ok(())
     }
 }
@@ -163,18 +161,38 @@ impl SqlPool {
         // initialize the sqlcipher key pragma
         let key_pragma = secure_write_key_pragma(dbk_secret)?;
 
-        // open a single write connection to the database
-        let mut write_con = rusqlite::Connection::open_with_flags(
-            &path,
-            OpenFlags::SQLITE_OPEN_READ_WRITE
-                | OpenFlags::SQLITE_OPEN_CREATE
-                | OpenFlags::SQLITE_OPEN_NO_MUTEX
-                | OpenFlags::SQLITE_OPEN_URI,
-        )
-        .map_err(one_err::OneErr::new)?;
-
-        // set generic pragmas
-        set_pragmas(&write_con, key_pragma.clone())?;
+        let mut write_con =
+            match create_configured_db_connection(&path, key_pragma.clone()) {
+                Ok(con) => con,
+                Err(
+                    err @ rusqlite::Error::SqliteFailure(
+                        rusqlite::ffi::Error {
+                            code: rusqlite::ffi::ErrorCode::NotADatabase,
+                            ..
+                        },
+                        ..,
+                    ),
+                ) => {
+                    if "true"
+                        == std::env::var("LAIR_MIGRATE_UNENCRYPTED")
+                            .unwrap_or_default()
+                            .as_str()
+                    {
+                        encrypt_unencrypted_database(
+                            &path,
+                            key_pragma.clone(),
+                        )?;
+                        create_configured_db_connection(
+                            &path,
+                            key_pragma.clone(),
+                        )
+                        .map_err(one_err::OneErr::new)?
+                    } else {
+                        return Err(one_err::OneErr::new(err));
+                    }
+                }
+                Err(e) => return Err(one_err::OneErr::new(e)),
+            };
 
         // only set WAL mode on the first write connection
         // it's a slow operation, and not needed on subsequent connections.
@@ -213,7 +231,8 @@ impl SqlPool {
             .map_err(one_err::OneErr::new)?;
 
             // set generic pragmas
-            set_pragmas(&read_con, key_pragma.clone())?;
+            set_pragmas(&read_con, key_pragma.clone())
+                .map_err(one_err::OneErr::new)?;
 
             *rc_mut = Some(read_con);
         }
@@ -289,6 +308,27 @@ impl SqlPool {
     }
 }
 
+fn create_configured_db_connection(
+    path: &std::path::PathBuf,
+    key_pragma: BufRead,
+) -> rusqlite::Result<rusqlite::Connection> {
+    use rusqlite::OpenFlags;
+
+    // open a single write connection to the database
+    let write_con = rusqlite::Connection::open_with_flags(
+        path,
+        OpenFlags::SQLITE_OPEN_READ_WRITE
+            | OpenFlags::SQLITE_OPEN_CREATE
+            | OpenFlags::SQLITE_OPEN_NO_MUTEX
+            | OpenFlags::SQLITE_OPEN_URI,
+    )?;
+
+    // set generic pragmas
+    set_pragmas(&write_con, key_pragma)?;
+
+    Ok(write_con)
+}
+
 impl AsLairStore for SqlPool {
     fn get_bidi_ctx_key(&self) -> sodoken::BufReadSized<32> {
         self.0.lock().ctx_secret.clone()
@@ -486,21 +526,90 @@ fn secure_write_key_pragma(
 fn set_pragmas(
     con: &rusqlite::Connection,
     key_pragma: BufRead,
-) -> LairResult<()> {
-    con.busy_timeout(std::time::Duration::from_millis(30_000))
-        .map_err(one_err::OneErr::new)?;
+) -> rusqlite::Result<()> {
+    con.busy_timeout(std::time::Duration::from_millis(30_000))?;
 
     con.execute_optional(
         std::str::from_utf8(&key_pragma.read_lock()).unwrap(),
         [],
     )?;
 
-    con.pragma_update(None, "trusted_schema", "0".to_string())
-        .map_err(one_err::OneErr::new)?;
+    con.pragma_update(None, "trusted_schema", "0".to_string())?;
+
+    con.pragma_update(None, "synchronous", "1".to_string())?;
 
-    con.pragma_update(None, "synchronous", "1".to_string())
+    Ok(())
+}
+
+fn encrypt_unencrypted_database(
+    path: &std::path::PathBuf,
+    key_pragma: BufRead,
+) -> LairResult<()> {
+    // e.g. keystore/store_file -> keystore/store_file-encrypted
+    let encrypted_path = path
+        .parent()
+        .ok_or_else(|| -> one_err::OneErr {
+            format!("Database file path has no parent: {:?}", path).into()
+        })?
+        .join(
+            path.file_stem()
+                .and_then(|s| s.to_str())
+                .ok_or_else(|| -> one_err::OneErr {
+                    format!("Database file path has no name: {:?}", path).into()
+                })?
+                .to_string()
+                + "-encrypted"
+                + &path
+                    .extension()
+                    .and_then(|s| s.to_str())
+                    .map(|p| ".".to_string() + p)
+                    .unwrap_or_default(),
+        );
+
+    tracing::warn!(
+        "Attempting encryption of unencrypted database: {:?} -> {:?}",
+        path,
+        encrypted_path
+    );
+
+    // Migrate the database
+    {
+        let conn =
+            rusqlite::Connection::open(path).map_err(one_err::OneErr::new)?;
+
+        // Ensure everything in the WAL is written to the main database
+        conn.execute("VACUUM", ()).map_err(one_err::OneErr::new)?;
+
+        // Start an exclusive transaction to avoid anybody writing to the database while we're migrating it
+        conn.execute("BEGIN EXCLUSIVE", ())
+            .map_err(one_err::OneErr::new)?;
+
+        let lock = key_pragma.read_lock();
+        conn.execute(
+            "ATTACH DATABASE :db_name AS encrypted KEY :key",
+            rusqlite::named_params! {
+                ":db_name": encrypted_path.to_str(),
+                ":key": &lock[14..81],
+            },
+        )
         .map_err(one_err::OneErr::new)?;
 
+        conn.query_row("SELECT sqlcipher_export('encrypted')", (), |_| Ok(0))
+            .map_err(one_err::OneErr::new)?;
+
+        conn.execute("COMMIT", ()).map_err(one_err::OneErr::new)?;
+
+        conn.execute("DETACH DATABASE encrypted", ())
+            .map_err(one_err::OneErr::new)?;
+        conn.close()
+            .map_err(|(_, err)| err)
+            .map_err(one_err::OneErr::new)?;
+    }
+
+    // Swap the databases over
+    std::fs::remove_file(path)?;
+    std::fs::rename(encrypted_path, path)?;
+
     Ok(())
 }
 

crates/lair_keystore/tests/common.rs

@@ -0,0 +1,48 @@
+use lair_keystore::dependencies::*;
+use lair_keystore_api::prelude::*;
+use std::sync::Arc;
+
+pub async fn create_config(
+    tmpdir: &tempdir::TempDir,
+    passphrase: sodoken::BufRead,
+) -> Arc<LairServerConfigInner> {
+    // create the config for the test server
+    Arc::new(
+        hc_seed_bundle::PwHashLimits::Minimum
+            .with_exec(|| {
+                LairServerConfigInner::new(tmpdir.path(), passphrase.clone())
+            })
+            .await
+            .unwrap(),
+    )
+}
+
+pub async fn connect_with_config(
+    config: Arc<LairServerConfigInner>,
+    passphrase: sodoken::BufRead,
+) -> LairResult<lair_keystore_api::LairClient> {
+    // execute the server
+    lair_keystore::server::StandaloneServer::new(config.clone())
+        .await?
+        .run(passphrase.clone())
+        .await?;
+
+    // create a client connection
+    lair_keystore_api::ipc_keystore::ipc_keystore_connect(
+        config.connection_url.clone(),
+        passphrase,
+    )
+    .await
+}
+
+#[allow(dead_code)]
+pub async fn connect(
+    tmpdir: &tempdir::TempDir,
+) -> lair_keystore_api::LairClient {
+    // set up a passphrase
+    let passphrase = sodoken::BufRead::from(&b"passphrase"[..]);
+
+    let config = create_config(tmpdir, passphrase.clone()).await;
+
+    connect_with_config(config, passphrase).await.unwrap()
+}