From f1ce3f466c3e30fa3c100ba7ca9547387474534c Mon Sep 17 00:00:00 2001 From: Piotr Orzechowski Date: Sat, 6 Apr 2024 08:32:15 +0200 Subject: [PATCH] Add redis tower session store --- .github/workflows/general.yml | 4 + Cargo.lock | 193 ++++++++++++++++++++++++++++++++++ Cargo.toml | 2 + configuration/base.yaml | 1 + scripts/init_db.sh | 2 +- scripts/init_redis.sh | 24 +++++ src/configuration.rs | 1 + src/main.rs | 2 +- src/startup.rs | 50 +++++++-- tests/api/helpers.rs | 4 +- 10 files changed, 273 insertions(+), 10 deletions(-) create mode 100755 scripts/init_redis.sh diff --git a/.github/workflows/general.yml b/.github/workflows/general.yml index 8a93ad6..c2b1aa6 100644 --- a/.github/workflows/general.yml +++ b/.github/workflows/general.yml @@ -30,6 +30,10 @@ jobs: POSTGRES_DB: postgres ports: - 5432:5432 + redis: + image: redis:7 + ports: + - 6379:6379 steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable diff --git a/Cargo.lock b/Cargo.lock index a973488..bf92e01 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -51,6 +51,12 @@ version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + [[package]] name = "argon2" version = "0.5.3" @@ -332,6 +338,16 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "cc" version = "1.0.90" @@ -434,6 +450,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "cookie-factory" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2" +dependencies = [ + "futures", +] + [[package]] name = "cookie_store" version = "0.20.0" @@ -491,6 +516,12 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crc16" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff" + [[package]] name = "crossbeam-queue" version = "0.3.11" @@ -667,6 +698,15 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + [[package]] name = "flume" version = "0.11.0" @@ -708,6 +748,32 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fred" +version = "8.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b8e3a1339ed45ad8fde94530c4bdcbd5f371a3c6bd3bf57682923792830aa37" +dependencies = [ + "arc-swap", + "async-trait", + "bytes", + "bytes-utils", + "crossbeam-queue", + "float-cmp", + "futures", + "log", + "parking_lot", + "rand", + "redis-protocol", + "semver", + "socket2", + "tokio", + "tokio-stream", + "tokio-util", + "url", + "urlencoding", +] + [[package]] name = "futures" version = "0.3.30" @@ -1215,6 +1281,7 @@ checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", + "serde", ] [[package]] @@ -1748,6 +1815,20 @@ dependencies = [ "rand_core", ] +[[package]] +name = "redis-protocol" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c31deddf734dc0a39d3112e73490e88b61a05e83e074d211f348404cee4d2c6" +dependencies = [ + "bytes", + "bytes-utils", + "cookie-factory", + "crc16", + "log", + "nom", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -1839,6 +1920,28 @@ dependencies = [ "winreg", ] +[[package]] +name = "rmp" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f9860a6cc38ed1da53456442089b4dfa35e7cedaa326df63017af88385e6b20" +dependencies = [ + "byteorder", + "num-traits", + "paste", +] + +[[package]] +name = "rmp-serde" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bffea85eea980d8a74453e5d02a8d93028f3c34725de143085a844ebe953258a" +dependencies = [ + "byteorder", + "rmp", + "serde", +] + [[package]] name = "ron" version = "0.8.1" @@ -1972,6 +2075,12 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" + [[package]] name = "serde" version = "1.0.197" @@ -2621,6 +2730,23 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower-cookies" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fd0118512cf0b3768f7fcccf0bef1ae41d68f2b45edc1e77432b36c97c56c6d" +dependencies = [ + "async-trait", + "axum-core", + "cookie 0.18.1", + "futures-util", + "http 1.1.0", + "parking_lot", + "pin-project-lite", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-http" version = "0.5.2" @@ -2652,6 +2778,71 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +[[package]] +name = "tower-sessions" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2defcf75e3a27d56238eee63fcc56cd2cafcc444f28ca3b41ecf147c6eb56e8d" +dependencies = [ + "async-trait", + "http 1.1.0", + "time", + "tokio", + "tower-cookies", + "tower-layer", + "tower-service", + "tower-sessions-core", + "tower-sessions-memory-store", + "tracing", +] + +[[package]] +name = "tower-sessions-core" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa0f238e4b39b7162564be8b2da85fc2a4d0d82027b5195c8f4c1a0ba7206c8" +dependencies = [ + "async-trait", + "axum-core", + "base64 0.22.0", + "futures", + "http 1.1.0", + "parking_lot", + "rand", + "serde", + "serde_json", + "thiserror", + "time", + "tokio", + "tracing", +] + +[[package]] +name = "tower-sessions-memory-store" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f261aa8acac419806c3bdef58ab735e4cee6837533dc85a6d899927e8b0d42bd" +dependencies = [ + "async-trait", + "time", + "tokio", + "tower-sessions-core", +] + +[[package]] +name = "tower-sessions-redis-store" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0460effd120251714fae61067fa3e8b2a3c8202501cdcae375d4bd14194c85cf" +dependencies = [ + "async-trait", + "fred", + "rmp-serde", + "thiserror", + "time", + "tower-sessions-core", +] + [[package]] name = "tracing" version = "0.1.40" @@ -3222,6 +3413,8 @@ dependencies = [ "tokio", "tower", "tower-http", + "tower-sessions", + "tower-sessions-redis-store", "tracing", "tracing-bunyan-formatter", "tracing-log 0.2.0", diff --git a/Cargo.toml b/Cargo.toml index 1c35eb9..39fbd61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,8 @@ time = { version = "0.3.34", features = ["macros", "serde"] } tokio = { version = "1.36.0", features = ["macros", "rt-multi-thread"] } tower = "0.4.13" tower-http = { version = "0.5.1", features = ["request-id", "trace", "util"] } +tower-sessions = "0.12.1" +tower-sessions-redis-store = "0.12.0" tracing = "0.1.40" tracing-bunyan-formatter = "0.3.9" tracing-log = "0.2.0" diff --git a/configuration/base.yaml b/configuration/base.yaml index c98d8b7..ddf152c 100644 --- a/configuration/base.yaml +++ b/configuration/base.yaml @@ -1,6 +1,7 @@ application: port: 8000 hmac_secret: long-and-very-secret-random-key-needed-to-verify-message-integrity + redis_uri: redis://localhost:6379 database: host: localhost port: 5432 diff --git a/scripts/init_db.sh b/scripts/init_db.sh index 3ba9956..5c0b55e 100755 --- a/scripts/init_db.sh +++ b/scripts/init_db.sh @@ -35,7 +35,7 @@ if [[ ${SKIP_PODMAN:=false} == false ]]; then --env POSTGRES_USER="${db_user}" \ --env POSTGRES_PASSWORD="${db_password}" \ --env POSTGRES_DB="${db_name}" \ - postgres \ + docker.io/library/postgres:latest \ postgres -N 1000 sleep 5 diff --git a/scripts/init_redis.sh b/scripts/init_redis.sh new file mode 100755 index 0000000..900ac81 --- /dev/null +++ b/scripts/init_redis.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -eou pipefail + +podman_cmd=('podman') +if [[ -f /run/.containerenv ]] && [[ -f /run/.toolboxenv ]]; then + podman_cmd=('flatpak-spawn' '--host' 'podman') +fi + +RUNNING_CONTAINER=$("${podman_cmd[@]}" ps --filter 'name=redis' --format '{{.ID}}') +if [[ -n $RUNNING_CONTAINER ]]; then + >&2 echo 'there is a redis container already running, kill it with' + >&2 echo " ${podman_cmd[@]} kill ${RUNNING_CONTAINER}" + exit 1 +fi + +"${podman_cmd[@]}" run \ + --rm \ + --detach \ + --name "redis_$(date '+%s')" \ + --publish 6379:6379 \ + docker.io/library/redis:latest + +>&2 echo 'Readis is ready to go!' diff --git a/src/configuration.rs b/src/configuration.rs index 83638b3..82b276e 100644 --- a/src/configuration.rs +++ b/src/configuration.rs @@ -23,6 +23,7 @@ pub struct ApplicationSettings { pub port: u16, pub base_url: String, pub hmac_secret: Secret, + pub redis_uri: Secret, } #[derive(Deserialize)] diff --git a/src/main.rs b/src/main.rs index 3347383..115cafb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,7 +5,7 @@ use zero2prod::{ }; #[tokio::main] -async fn main() -> Result<(), std::io::Error> { +async fn main() -> Result<(), anyhow::Error> { let subscriber = get_subscriber("zero2prod".into(), "info".into(), std::io::stdout); init_subscriber(subscriber); diff --git a/src/startup.rs b/src/startup.rs index 7b9d16d..a15f3f4 100644 --- a/src/startup.rs +++ b/src/startup.rs @@ -1,27 +1,39 @@ use crate::{ app_state::AppState, - configuration::{DatabaseSettings, Settings}, + configuration::{ApplicationSettings, DatabaseSettings, Settings}, email_client::EmailClient, request_id::RequestUuid, routes::{health_check, home, login, newsletters, subscriptions, subscriptions_confirm}, telemetry::request_span, }; +use anyhow::anyhow; use axum::{http::Uri, serve::Serve, Router}; use axum_extra::extract::cookie::Key; use secrecy::{ExposeSecret, Secret}; use sqlx::{postgres::PgPoolOptions, PgPool}; use std::{net::SocketAddr, str::FromStr}; +use time::Duration; use tokio::net::TcpListener; use tower::ServiceBuilder; use tower_http::{ trace::{DefaultOnRequest, DefaultOnResponse, TraceLayer}, ServiceBuilderExt, }; +use tower_sessions::{Expiry, SessionManagerLayer}; +use tower_sessions_redis_store::{ + fred::{ + clients::RedisPool, + interfaces::ClientLike, + types::{ConnectHandle, RedisConfig}, + }, + RedisStore, +}; use tracing::Level; pub struct Application { local_addr: SocketAddr, server: Serve, + redis_conn: ConnectHandle, } impl Application { @@ -32,7 +44,7 @@ impl Application { .await .expect("Failed to open listener"); - let db_pool = get_connection_pool(&config.database); + let db_pool = get_pg_connection_pool(&config.database); let sender_email = config .email_client @@ -47,6 +59,8 @@ impl Application { timeout, ); + let (redis_pool, redis_conn) = get_redis_connection_pool(&config.application).await; + let local_addr = listener .local_addr() .expect("Failed to get local address from the listener"); @@ -57,32 +71,52 @@ impl Application { email_client, config.application.base_url, config.application.hmac_secret, + redis_pool, ) .await; - Self { local_addr, server } + Self { + local_addr, + server, + redis_conn, + } } pub fn local_addr(&self) -> SocketAddr { self.local_addr } - pub async fn run_until_stopped(self) -> Result<(), std::io::Error> { + pub async fn run_until_stopped(self) -> Result<(), anyhow::Error> { tracing::info!("Listening on {}", self.local_addr); - self.server.await + self.server.await?; + self.redis_conn.await?.map_err(|e| anyhow!(e)) } } -pub fn get_connection_pool(config: &DatabaseSettings) -> PgPool { +pub fn get_pg_connection_pool(config: &DatabaseSettings) -> PgPool { PgPoolOptions::new().connect_lazy_with(config.with_db()) } +async fn get_redis_connection_pool(config: &ApplicationSettings) -> (RedisPool, ConnectHandle) { + let config = RedisConfig::from_url(config.redis_uri.expose_secret()) + .expect("Failed to create redis config"); + let pool = RedisPool::new(config, None, None, None, 6).expect("Failed to create redis pool"); + let conn = pool.connect(); + + pool.wait_for_connect() + .await + .expect("Failed to connect redis clients"); + + (pool, conn) +} + async fn run( listener: TcpListener, db_pool: PgPool, email_client: EmailClient, base_url: String, hmac_secret: Secret, + redis_pool: RedisPool, ) -> Serve { let app_state = AppState { db_pool, @@ -99,6 +133,10 @@ async fn run( .merge(home::router()) .merge(login::router()) .with_state(app_state) + .layer( + SessionManagerLayer::new(RedisStore::new(redis_pool)) + .with_expiry(Expiry::OnInactivity(Duration::seconds(10))), + ) .layer( ServiceBuilder::new() .set_x_request_id(RequestUuid) diff --git a/tests/api/helpers.rs b/tests/api/helpers.rs index 4951401..24f30cf 100644 --- a/tests/api/helpers.rs +++ b/tests/api/helpers.rs @@ -9,7 +9,7 @@ use uuid::Uuid; use wiremock::MockServer; use zero2prod::{ configuration::{get_configuration, DatabaseSettings}, - startup::{get_connection_pool, Application}, + startup::{get_pg_connection_pool, Application}, telemetry::{get_subscriber, init_subscriber}, }; @@ -202,7 +202,7 @@ async fn configure_database(configuration: &DatabaseSettings) -> PgPool { .await .expect("Failed to create database"); - let pool = get_connection_pool(configuration); + let pool = get_pg_connection_pool(configuration); sqlx::migrate!("./migrations") .run(&pool)