Simple HTTP Proxy (#49)

* draft proxy

* fix proxy

* Decompress response before recording & remove header on forwarding to requester

Signed-off-by: Alexander Ignition <izh.sever@gmail.com>

* Update AppConfiguration

* Update ProxyMiddleware

* Update Loggers.swift

Add MultiplexLogHandler

* Add text proxy

* Update README.md

* Update Proxy example in README.md

Co-authored-by: Anton Glezman <a.glezman@redmadrobot.com>

Co-authored-by: Vasily Fedorov <vasily.fedorov@alibaba-inc.com>
Co-authored-by: Anton Glezman <a.glezman@redmadrobot.com>

Packages/CatbirdApp/Sources/CatbirdApp/AppConfiguration.swift

@@ -1,20 +1,17 @@
-import Foundation
+import Vapor
 
 /// Application configuration.
 public struct AppConfiguration {
 
-    /// Application work mode.
-    public enum Mode: Equatable {
-        case write(URL)
-        case read
-    }
+    public let isRecordMode: Bool
 
-    /// Application work mode.
-    public let mode: Mode
+    public let proxyEnabled: Bool
 
     /// The directory for mocks.
     public let mocksDirectory: URL
 
+    public let redirectUrl: URL?
+
     public let maxBodySize: String
 }
 
@@ -38,11 +35,16 @@ extension AppConfiguration {
             return url
         }()
 
+        let isRecordMode = environment["CATBIRD_RECORD_MODE"].flatMap { NSString(string: $0).boolValue } ?? false
+        let proxyEnabled = environment["CATBIRD_PROXY_ENABLED"].flatMap { NSString(string: $0).boolValue } ?? false
+        let redirectUrl = environment["CATBIRD_REDIRECT_URL"].flatMap { URL(string: $0) }
         let maxBodySize = environment["CATBIRD_MAX_BODY_SIZE", default: "50mb"]
 
-        if let path = environment["CATBIRD_PROXY_URL"], let url = URL(string: path) {
-            return AppConfiguration(mode: .write(url), mocksDirectory: mocksDirectory, maxBodySize: maxBodySize)
-        }
-        return AppConfiguration(mode: .read, mocksDirectory: mocksDirectory, maxBodySize: maxBodySize)
+        return AppConfiguration(
+            isRecordMode: isRecordMode,
+            proxyEnabled: proxyEnabled,
+            mocksDirectory: mocksDirectory,
+            redirectUrl: redirectUrl,
+            maxBodySize: maxBodySize)
     }
 }

Packages/CatbirdApp/Sources/CatbirdApp/Common/FileDirectoryPath.swift

@@ -9,7 +9,7 @@ struct FileDirectoryPath {
     }
 
     func preferredFileURL(for request: Request) -> URL {
-        var fileUrl = url.appendingPathComponent(request.url.string)
+        var fileUrl = fileURL(for: request)
 
         guard fileUrl.pathExtension.isEmpty else {
             return fileUrl
@@ -21,7 +21,7 @@ struct FileDirectoryPath {
     }
 
     func filePaths(for request: Request) -> [String] {
-        let fileUrl = url.appendingPathComponent(request.url.string)
+        let fileUrl = fileURL(for: request)
 
         var urls: [URL] = []
         if fileUrl.pathExtension.isEmpty {
@@ -32,4 +32,16 @@ struct FileDirectoryPath {
         urls.append(fileUrl)
         return urls.map { $0.absoluteString }
     }
+
+    private func fileURL(for request: Request) -> URL {
+        var fileUrl = url
+        if let host = request.url.host {
+            fileUrl.appendPathComponent(host)
+        }
+        fileUrl.appendPathComponent(request.url.path)
+        if fileUrl.absoluteString.hasSuffix("/") {
+            fileUrl.appendPathComponent("index")
+        }
+        return fileUrl
+    }
 }

Packages/CatbirdApp/Sources/CatbirdApp/Common/HTTPClientSupport.swift

@@ -0,0 +1,43 @@
+import Vapor
+
+extension Request {
+    /// Send HTTP request.
+    ///
+    /// - Parameter configure: client request configuration function.
+    /// - Returns: Server response.
+    func send(configure: ((inout ClientRequest) -> Void)? = nil) -> EventLoopFuture<Response> {
+        return body
+            .collect(max: nil)
+            .flatMap { (bytesBuffer: ByteBuffer?) -> EventLoopFuture<Response> in
+                var clientRequest = self.clientRequest(body: bytesBuffer)
+                configure?(&clientRequest)
+                return self.client.send(clientRequest).map { (clientResponse: ClientResponse) -> Response in
+                    clientResponse.response(version: self.version)
+                }
+            }
+    }
+
+    /// Convert to HTTP client request.
+    private func clientRequest(body: ByteBuffer?) -> ClientRequest {
+        var headers = self.headers
+        if let host = headers.first(name: "Host") {
+            headers.replaceOrAdd(name: "X-Forwarded-Host", value: host)
+            headers.remove(name: "Host")
+        }
+        return ClientRequest(method: method, url: url, headers: headers, body: body)
+    }
+}
+
+extension HTTPHeaders {
+    fileprivate var contentLength: Int? {
+        first(name: "Content-Length").flatMap { Int($0) }
+    }
+}
+
+extension ClientResponse {
+    /// Convert to Server Response.
+    fileprivate func response(version: HTTPVersion) -> Response {
+        let body = body.map { Response.Body(buffer: $0) } ?? .empty
+        return Response(status: status, version: version, headers: headers, body: body)
+    }
+}

Packages/CatbirdApp/Sources/CatbirdApp/Common/Loggers.swift

@@ -13,7 +13,10 @@ enum Loggers {
         return Logging.Logger(label: CatbirdInfo.current.domain)
 #else
         return Logging.Logger(label: CatbirdInfo.current.domain) {
-            OSLogHandler(subsystem: $0, category: category)
+            Logging.MultiplexLogHandler([
+                OSLogHandler(subsystem: $0, category: category),
+                Logging.StreamLogHandler.standardOutput(label: $0)
+            ])
         }
 #endif
     }

Packages/CatbirdApp/Sources/CatbirdApp/Middlewares/AnyMiddleware.swift

@@ -2,11 +2,11 @@ import Vapor
 
 final class AnyMiddleware: Middleware {
 
-    private typealias Handler = (Request, Responder) -> EventLoopFuture<Response>
+    typealias Handler = (Request, Responder) -> EventLoopFuture<Response>
 
     private let handler: Handler
 
-    private init(handler: @escaping Handler) {
+    init(handler: @escaping Handler) {
         self.handler = handler
     }
 
@@ -24,19 +24,9 @@ extension AnyMiddleware {
     /// - Returns: A new `Middleware`.
     static func notFound(_ handler: @escaping (Request) -> EventLoopFuture<Response>) -> Middleware {
         AnyMiddleware { (request, responder) -> EventLoopFuture<Response> in
-            responder.respond(to: request)
-                .flatMap { (response: Response) -> EventLoopFuture<Response> in
-                    if response.status == .notFound {
-                        return handler(request)
-                    }
-                    return request.eventLoop.makeSucceededFuture(response)
-                }
-                .flatMapError { (error: Error) -> EventLoopFuture<Response> in
-                    if let abort = error as? AbortError, abort.status == .notFound {
-                        return handler(request)
-                    }
-                    return request.eventLoop.makeFailedFuture(error)
-                }
+            responder.respond(to: request).notFound {
+                handler(request)
+            }
         }
     }
 
@@ -49,3 +39,23 @@ extension AnyMiddleware {
     }
 
 }
+
+extension EventLoopFuture where Value: Response {
+    func notFound(
+        _ handler: @escaping () -> EventLoopFuture<Response>
+    ) -> EventLoopFuture<Response> {
+
+        return flatMap { [eventLoop] (response: Response) -> EventLoopFuture<Response> in
+            if response.status == .notFound {
+                return handler()
+            }
+            return eventLoop.makeSucceededFuture(response)
+        }
+        .flatMapError { [eventLoop] (error: Error) -> EventLoopFuture<Response> in
+            if let abort = error as? AbortError, abort.status == .notFound {
+                return handler()
+            }
+            return eventLoop.makeFailedFuture(error)
+        }
+    }
+}

Packages/CatbirdApp/Sources/CatbirdApp/Middlewares/ProxyMiddleware.swift

@@ -0,0 +1,24 @@
+import Vapor
+
+final class ProxyMiddleware: Middleware {
+
+    func respond(to request: Request, chainingTo next: Responder) -> EventLoopFuture<Response> {
+        if request.url.host == nil {
+            request.logger.info("Proxy break \(request.method) \(request.url)")
+            return next.respond(to: request)
+        }
+        return next.respond(to: request).notFound {
+            var url = request.url
+            if url.scheme == nil {
+                url.scheme = url.port == 443 ? "https" : "http"
+            }
+
+            request.logger.info("Proxy \(request.method) \(url), scheme \(url.scheme ?? "<nil>")")
+
+            // Send request to real host
+            return request.send {
+                $0.url = url
+            }
+        }
+    }
+}

Packages/CatbirdApp/Sources/CatbirdApp/Middlewares/RedirectMiddleware.swift

@@ -11,29 +11,17 @@ final class RedirectMiddleware: Middleware {
     // MARK: - Middleware
 
     func respond(to request: Request, chainingTo next: Responder) -> EventLoopFuture<Response> {
-        return request.body.collect(max: nil).flatMap { (body: ByteBuffer?) -> EventLoopFuture<Response> in
-            var headers = request.headers
-            headers.remove(name: "Host")
-
-            var clientRequest = ClientRequest(
-                method: request.method,
-                url: self.redirectURI,
-                headers: headers,
-                body: request.body.data)
+        // Handle only direct requests to catbird
+        if request.url.host != nil {
+            return next.respond(to: request) // proxy request
+        }
 
-            clientRequest.url.string += request.url.string
+        var uri = redirectURI
+        uri.string += request.url.string
 
-            return request
-                .client
-                .send(clientRequest)
-                .map { (response: ClientResponse) -> Response in
-                    let body = response.body.map { Response.Body(buffer: $0) } ?? .empty
-                    return Response(
-                        status: response.status,
-                        version: request.version,
-                        headers: response.headers,
-                        body: body)
-                }
+        // Send request to redirect host
+        return request.send {
+            $0.url = uri
         }
     }
 }

Packages/CatbirdApp/Sources/CatbirdApp/configure.swift

@@ -1,5 +1,6 @@
 import CatbirdAPI
 import Vapor
+import NIOSSL
 
 public struct CatbirdInfo: Content {
     public static let current = CatbirdInfo(
@@ -28,26 +29,38 @@ public func configure(_ app: Application, _ configuration: AppConfiguration) thr
         store: InMemoryResponseStore(),
         logger: Loggers.inMemoryStore)
 
-    // MARK: - Register Middlewares
+    // MARK: - Register Middleware
 
     // Pubic resource for web page
     app.middleware.use(FileMiddleware(publicDirectory: app.directory.publicDirectory))
-    switch configuration.mode {
-    case .read:
-        app.logger.info("Read mode")
-        // try read from static mocks if route not found
-        app.middleware.use(AnyMiddleware.notFound(fileStore.response))
-        // try read from dynamic mocks
-        app.middleware.use(AnyMiddleware.notFound(inMemoryStore.response))
-    case .write(let url):
-        app.logger.info("Write mode")
+    if configuration.isRecordMode {
+        app.logger.info("Record mode")
+        app.http.client.configuration.decompression = .enabled(limit: .none)
         // capture response and write to file
         app.middleware.use(AnyMiddleware.capture { request, response in
+            if response.headers.contains(name: "Content-encoding") {
+                response.headers.remove(name: "Content-encoding")
+            }
             let pattern = RequestPattern(method: .init(request.method.rawValue), url: request.url.string)
             let mock = ResponseMock(status: Int(response.status.code), body: response.body.data)
             return fileStore.perform(.update(pattern, mock), for: request).map { _ in response }
         })
-        // redirect request to another server
+        // catch 404 and try read from real server
+        if configuration.proxyEnabled {
+            app.middleware.use(ProxyMiddleware())
+        }
+    } else {
+        app.logger.info("Read mode")
+        // catch 404 and try read from real server
+        if configuration.proxyEnabled {
+            app.middleware.use(ProxyMiddleware())
+        }
+        // try read from static mocks if route not found
+        app.middleware.use(AnyMiddleware.notFound(fileStore.response))
+        // try read from dynamic mocks
+        app.middleware.use(AnyMiddleware.notFound(inMemoryStore.response))
+    }
+    if let url = configuration.redirectUrl {
         app.middleware.use(RedirectMiddleware(serverURL: url))
     }
 

Packages/CatbirdApp/Tests/CatbirdAppTests/App/AppConfigurationTests.swift

@@ -5,18 +5,24 @@ final class AppConfigurationTests: XCTestCase {
 
     func testDetectReadMode() throws {
         let config = try AppConfiguration.detect(from: [:])
-        XCTAssertEqual(config.mode, .read)
+        XCTAssertEqual(config.isRecordMode, false)
+        XCTAssertEqual(config.proxyEnabled, false)
         XCTAssertEqual(config.mocksDirectory.absoluteString, AppConfiguration.sourceDir)
+        XCTAssertNil(config.redirectUrl)
         XCTAssertEqual(config.maxBodySize, "50mb")
     }
 
     func testDetectWriteMode() throws {
         let config = try AppConfiguration.detect(from: [
-            "CATBIRD_PROXY_URL": "/",
+            "CATBIRD_RECORD_MODE": "1",
+            "CATBIRD_PROXY_ENABLED": "1",
+            "CATBIRD_REDIRECT_URL": "https://example.com",
             "CATBIRD_MAX_BODY_SIZE": "1kb"
         ])
-        XCTAssertEqual(config.mode, .write(URL(string: "/")!))
+        XCTAssertEqual(config.isRecordMode, true)
+        XCTAssertEqual(config.proxyEnabled, true)
         XCTAssertEqual(config.mocksDirectory.absoluteString, AppConfiguration.sourceDir)
+        XCTAssertEqual(config.redirectUrl?.absoluteString, "https://example.com")
         XCTAssertEqual(config.maxBodySize, "1kb")
     }
 

Packages/CatbirdApp/Tests/CatbirdAppTests/App/AppTestCase.swift

@@ -9,18 +9,24 @@ class AppTestCase: XCTestCase {
         willSet { app?.shutdown() }
     }
 
-    func setUpApp(mode: AppConfiguration.Mode) throws {
+    func setUpApp(
+        isRecordMode: Bool = false,
+        proxyEnabled: Bool = false,
+        redirectUrl: URL? = nil
+    ) throws {
         let config = AppConfiguration(
-            mode: mode,
+            isRecordMode: isRecordMode,
+            proxyEnabled: proxyEnabled,
             mocksDirectory: URL(string: mocksDirectory)!,
+            redirectUrl: redirectUrl,
             maxBodySize: "50kb")
         app = Application(.testing)
         try configure(app, config)
     }
 
     override func setUp() {
         super.setUp()
-        XCTAssertNoThrow(try setUpApp(mode: .read))
+        XCTAssertNoThrow(try setUpApp())
         XCTAssertEqual(app.routes.defaultMaxBodySize, 51200)
     }