Fixes, upgrades

Upgrades core deps. Enables HTTP/2 on the proxy front-end, but backend fulfillment is still HTTP/1.x. Strips all compression methods from the front end. Configures Kestrel for optimal throughput. Ensures that headers are replaced if they exist. This ensures that client changes always apply. Removes arbitrary limits, such as max length on request strings. Fixes an issue where ASP.NET Core incorrectly decodes some URL's, which causes some things to randomly fail as bad requests. Google maps is an example of what suffered in previous versions. Now builds out the full URL based on the raw values sent by the browser. This applies to websockets as well. Adds the ability to inspect individual websocket messages. Fixes an issue where the whole body inspection callback was invoked on websockets even if not requested. Exempts well-defined windows-exclusive ports from packet interception. Everything is now extremely fast, and stable. No more public changes or additions will be made.
TechnikEmpire · Dec 11, 2018 · ae0025c · ae0025c
1 parent ee34d47
commit ae0025c
Show file tree

Hide file tree

Showing 8 changed files with 234 additions and 37 deletions.
diff --git a/CitadelCore/CitadelCore.csproj b/CitadelCore/CitadelCore.csproj
@@ -12,14 +12,23 @@
  <PackageIconUrl />
  <RepositoryUrl>https://github.com/TechnikEmpire/CitadelCore</RepositoryUrl>
  <PackageTags>proxy filtering content-filtering transparent-proxy</PackageTags>
- <PackageReleaseNotes>Includes originating message info in replay callback. This is a breaking API change, so version is bumped to 4.x.
-Fixes an issue where websockets were malfunctioning as a result of the new websocket fork work.</PackageReleaseNotes>
+ <PackageReleaseNotes>Upgrades core deps.
+Enables HTTP/2 on the proxy front-end, but backend fulfillment is still HTTP/1.x.
+Strips all compression methods from the front end.
+Configures Kestrel for optimal throughput.
+Ensures that headers are replaced if they exist. This ensures that client changes always apply.
+Removes arbitrary limits, such as max length on request strings.
+Fixes an issue where ASP.NET Core incorrectly decodes some URL's, which causes some things to randomly fail as bad requests. Google maps is an example of what suffered in previous versions.
+Now builds out the full URL based on the raw values sent by the browser. This applies to websockets as well.
+Adds the ability to inspect individual websocket messages.
+Fixes an issue where the whole body inspection callback was invoked on websockets even if not requested.
+Everything is now extremely fast, and stable. No more public changes or additions will be made.</PackageReleaseNotes>
  <Title>CitadelCore</Title>
  <Summary>Transparent filtering HTTP/S and Websocket/WebsocketSecure proxy.</Summary>
  <Description>Transparent filtering HTTP/S and Websocket/WebsocketSecure proxy.</Description>
- <Version>4.0.1</Version>
- <AssemblyVersion>4.0.1.0</AssemblyVersion>
- <FileVersion>4.0.1.0</FileVersion> 
+ <Version>4.2.3</Version>
+ <AssemblyVersion>4.2.3.0</AssemblyVersion>
+ <FileVersion>4.2.3.0</FileVersion> 
  </PropertyGroup>
 
  <ItemGroup Label="dotnet pack instructions">
@@ -62,9 +71,8 @@ Fixes an issue where websockets were malfunctioning as a result of the new webso
 
  <ItemGroup>
  <PackageReference Include="CitadelCore.Websockets.Client.Managed" Version="1.1.1" />
- <PackageReference Include="Microsoft.AspNetCore.Diagnostics" Version="2.1.1" />
- <PackageReference Include="Microsoft.AspNetCore.ResponseCompression" Version="2.1.1" />
- <PackageReference Include="Microsoft.AspNetCore.Server.Kestrel" Version="2.1.3" />
+ <PackageReference Include="Microsoft.AspNetCore.Diagnostics" Version="2.2.0" />
+ <PackageReference Include="Microsoft.AspNetCore.Server.Kestrel" Version="2.2.0" />
  <PackageReference Include="Microsoft.AspNetCore.WebSockets.Server" Version="0.1.0" />
  <PackageReference Include="Portable.BouncyCastle" Version="1.8.4" />
  <PackageReference Include="StreamExtended" Version="[1.0.81]" />

diff --git a/CitadelCore/Extensions/HttpRequestExtensions.cs b/CitadelCore/Extensions/HttpRequestExtensions.cs
@@ -54,6 +54,11 @@ public static NameValueCollection PopulateHeaders(this HttpRequest message, Name
 
  try
  {
+ if (message.Headers.ContainsKey(key))
+ {
+ message.Headers.Remove(key);
+ }
+
  message.Headers.Add(key, headers.GetValues(key));
  clonedCollection.Remove(key);
  }

diff --git a/CitadelCore/Extensions/HttpRequestMessageExtensions.cs b/CitadelCore/Extensions/HttpRequestMessageExtensions.cs
@@ -50,7 +50,12 @@ public static NameValueCollection PopulateHeaders(this HttpRequestMessage messag
  {
  continue;
  }
-
+
+ if (message.Headers.Contains(key))
+ {
+ message.Headers.Remove(key);
+ }
+
  if (message.Headers.TryAddWithoutValidation(key, headers.GetValues(key)))
  { 
  clonedCollection.Remove(key);
@@ -59,6 +64,11 @@ public static NameValueCollection PopulateHeaders(this HttpRequestMessage messag
  {
  if (message.Content != null)
  {
+ if (message.Content.Headers.Contains(key))
+ {
+ message.Content.Headers.Remove(key);
+ }
+
  if (message.Content.Headers.TryAddWithoutValidation(key, headers.GetValues(key)))
  {
  clonedCollection.Remove(key);

diff --git a/CitadelCore/Extensions/HttpResponseExtensions.cs b/CitadelCore/Extensions/HttpResponseExtensions.cs
@@ -54,6 +54,11 @@ public static NameValueCollection PopulateHeaders(this HttpResponse message, Nam
 
  try
  {
+ if (message.Headers.ContainsKey(key))
+ {
+ message.Headers.Remove(key);
+ }
+
  message.Headers.Add(key, new Microsoft.Extensions.Primitives.StringValues(headers.GetValues(key)));
  clonedCollection.Remove(key);
  }

diff --git a/CitadelCore/Extensions/HttpResponseMessageExtensions.cs b/CitadelCore/Extensions/HttpResponseMessageExtensions.cs
@@ -51,6 +51,11 @@ public static NameValueCollection PopulateHeaders(this HttpResponseMessage messa
  continue;
  }
 
+ if (message.Headers.Contains(key))
+ {
+ message.Headers.Remove(key);
+ }
+
  if (message.Headers.TryAddWithoutValidation(key, headers.GetValues(key)))
  {
  clonedCollection.Remove(key);
@@ -59,6 +64,11 @@ public static NameValueCollection PopulateHeaders(this HttpResponseMessage messa
  {
  if (message.Content != null)
  {
+ if (message.Content.Headers.Contains(key))
+ {
+ message.Content.Headers.Remove(key);
+ }
+
  if (message.Content.Headers.TryAddWithoutValidation(key, headers.GetValues(key)))
  {
  clonedCollection.Remove(key);

diff --git a/CitadelCore/Net/Handlers/FilterHttpResponseHandler.cs b/CitadelCore/Net/Handlers/FilterHttpResponseHandler.cs
@@ -11,14 +11,18 @@
 using CitadelCore.Net.Handlers.Replay;
 using CitadelCore.Net.Http;
 using CitadelCore.Net.Proxy;
+using CitadelCore.Util;
 using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Http.Features;
 using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Net;
 using System.Net.Http;
+using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
+using System.Web;
 
 namespace CitadelCore.Net.Handlers
 {
@@ -114,15 +118,27 @@ public override async Task Handle(HttpContext context)
  HttpRequestMessage requestMsg = null;
 
  HttpClient upstreamClient = _client;
-
+ 
  try
  {
  // Use helper to get the full, proper URL for the request. var fullUrl = Microsoft.AspNetCore.Http.Extensions.UriHelper.GetDisplayUrl(context.Request);
- var fullUrl = Microsoft.AspNetCore.Http.Extensions.UriHelper.GetDisplayUrl(context.Request);
+ //var fullUrl = Microsoft.AspNetCore.Http.Extensions.UriHelper.GetDisplayUrl(context.Request);
+
+ var connFeature = context.Features.Get<IHttpRequestFeature>();
+
+ string fullUrl = string.Empty;
+
+ if (connFeature != null && connFeature.RawTarget != null && !string.IsNullOrEmpty(connFeature.RawTarget) && !(string.IsNullOrWhiteSpace(connFeature.RawTarget)))
+ {
+ fullUrl = $"{context.Request.Scheme}://{context.Request.Host}{connFeature.RawTarget}";
+ }
+ else
+ {
+ fullUrl = $"{context.Request.Scheme}://{context.Request.Host}{context.Request.Path}{context.Request.QueryString}";
+ }
 
  // Next we need to try and parse the URL as a URI, because the web client requires
  // this for connecting upstream.
-
  if (!Uri.TryCreate(fullUrl, UriKind.RelativeOrAbsolute, out Uri reqUrl))
  {
  LoggerProxy.Default.Error("Failed to parse HTTP URL.");
@@ -197,12 +213,17 @@ public override async Task Handle(HttpContext context)
 
  // Create the message AFTER we give the user a chance to alter things.
  requestMsg = new HttpRequestMessage(requestMessageNfo.Method, requestMessageNfo.Url);
+
  var initialFailedHeaders = requestMsg.PopulateHeaders(requestMessageNfo.Headers, requestMessageNfo.ExemptedHeaders);
 
  // Ensure that we match the protocol of the client!
  if (upstreamReqVersionMatch != null)
  {
- requestMsg.Version = upstreamReqVersionMatch;
+ // Upstream won't have HTTP/2 support on all OS versions!
+ if (upstreamReqVersionMatch.Major <= 1)
+ {
+ requestMsg.Version = upstreamReqVersionMatch;
+ }
  }
 
  // Check if we have a request body.
@@ -356,7 +377,7 @@ public override async Task Handle(HttpContext context)
  try
  {
  try
- {
+ { 
  response = await upstreamClient.SendAsync(requestMsg, HttpCompletionOption.ResponseHeadersRead, context.RequestAborted);
  }
  catch (Exception e)