compute canonical uri based on AWS doc

CHANGELOG.md

@@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Remove deprecation warnings due to usage of `utcnow` and `utcfromtimestamp`. Thanks to [`Raphael Krupinski`](https://github.com/rafalkrupinski).
 - `httpx_auth.AWS4Auth.default_include_headers` value kept growing in size every time a new `httpx_auth.AWS4Auth` instance was created with `security_token` parameter provided. Thanks to [`Miikka Koskinen`](https://github.com/miikka).
 - `httpx_auth.AWS4Auth` is now processing included headers without spaces in value faster.
-- `httpx_auth.AWS4Auth` canonical query string generation is now based entirely on AWS documentation, solving bugs in the original implementation from `requests-aws4auth`.
+- `httpx_auth.AWS4Auth` is now based almost entirely on AWS documentation, solving bugs in the original implementation from `requests-aws4auth`.
   - As the AWS documentation might be wrong or not exhaustive enough, feel free to open issues, should you encounter edge cases.
 
 ### Changed

README.md

@@ -692,7 +692,8 @@ Note that the following changes were made compared to `requests-aws4auth`:
   - It is not possible to provide an `AWSSigningKey` instance, use explicit parameters instead.
   - It is not possible to provide `raise_invalid_date` parameter anymore as the date will always be valid.
   - `host` is not considered as a specific Amazon service anymore (no test specific code).
-  - Canonical query string computation is based on AWS documentation and excludes fragment (`#` and following characters).
+  - Canonical query string computation is entirely based on AWS documentation (and consider undocumented fragment (`#` and following characters) as not part of the query string).
+  - Canonical uri computation is entirely based on AWS documentation.
 
 ### Parameters
 

httpx_auth/aws.py

@@ -3,15 +3,15 @@
 https://github.com/sam-washington/requests-aws4auth
 """
 
-import hmac
+import datetime
 import hashlib
-import posixpath
+import hmac
 import re
 import shlex
-import datetime
 from collections import defaultdict
-from urllib.parse import urlparse, quote, unquote
+from posixpath import normpath
 from typing import Generator, Tuple
+from urllib.parse import quote
 
 import httpx
 
@@ -100,7 +100,7 @@ def _canonical_request(
         return "\n".join(
             [
                 req.method.upper(),
-                self._canonical_uri(req.url),
+                self._canonical_uri(req.url, is_s3=self.service.lower() == "s3"),
                 self._canonical_query_string(req.url),
                 canonical_headers,
                 signed_headers,
@@ -140,26 +140,50 @@ def _string_to_sign(req: httpx.Request, canonical_request: str, scope: str) -> s
             ["AWS4-HMAC-SHA256", req.headers["x-amz-date"], scope, hsh.hexdigest()]
         )
 
-    def _canonical_uri(self, url: httpx.URL) -> str:
+    @staticmethod
+    def _canonical_uri(url: httpx.URL, is_s3: bool) -> str:
         """
-        Not documented anywhere, determined from aws4_testsuite examples,
-        problem reports and testing against the live services.
+        CanonicalURI is the URI-encoded version of the absolute path component of the URI
+        — everything starting with the "/" that follows the domain name and
+        up to the end of the string
+        or to the question mark character ('?') if you have query string parameters.
+
+        The URI in the following example, /examplebucket/myphoto.jpg, is the absolute path, and you don't encode the "/" in the absolute path:
+
+        http://s3.amazonaws.com/examplebucket/myphoto.jpg
+        >>> AWS4Auth._canonical_uri(httpx.URL("http://s3.amazonaws.com/examplebucket/myphoto.jpg"), is_s3=False)
+        '/examplebucket/myphoto.jpg'
+
+        Note
+        You do not normalize URI paths for requests to Amazon S3.
+        For example, you may have a bucket with an object named "my-object//example//photo.user".
+        Normalizing the path changes the object name in the request to "my-object/example/photo.user".
+        This is an incorrect path for that object.
+        >>> AWS4Auth._canonical_uri(httpx.URL("http://s3.amazonaws.com/my-object//example//photo.user"), is_s3=False)
+        '/my-object/example/photo.user'
+        >>> AWS4Auth._canonical_uri(httpx.URL("http://s3.amazonaws.com/my-object//example//photo.user"), is_s3=True)
+        '/my-object//example//photo.user'
+
+        Some limitation that should be covered but not documented by AWS:
+        - Trailing / should be kept
+        >>> AWS4Auth._canonical_uri(httpx.URL("http://s3.amazonaws.com/resource/"), is_s3=False)
+        '/resource/'
+
+        - Starting with // should be normalized
+        >>> AWS4Auth._canonical_uri(httpx.URL("http://s3.amazonaws.com//resource/"), is_s3=False)
+        '/resource/'
         """
-        url_str = str(url)
-        url = urlparse(url_str)
-        path = url.path
-        if len(path) == 0:
-            path = "/"
-        fixed_path = posixpath.normpath(path)
-        # Prevent multi /
-        fixed_path = re.sub("/+", "/", fixed_path)
-        if path.endswith("/") and not fixed_path.endswith("/"):
-            fixed_path += "/"
-        full_path = fixed_path
-        # S3 seems to require unquoting first.
-        if self.service == "s3":
-            full_path = unquote(full_path)
-        return quote(full_path, safe="/~")
+        resource = url.path
+        if not is_s3:
+            # Convert to absolute path until python provides a clean RFC implementation of path-absolute
+            absolute_path = normpath(resource)
+            if absolute_path.startswith("//"):
+                absolute_path = resource[1:]
+            if resource.endswith("/") and not absolute_path.endswith("/"):
+                absolute_path += "/"
+            resource = absolute_path
+
+        return uri_encode(resource, is_key=True)
 
     @staticmethod
     def _canonical_query_string(url: httpx.URL) -> str:

tests/aws_signature_v4/test_aws4auth_async.py

@@ -800,7 +800,7 @@ async def test_aws_auth_path_quoting(httpx_mock: HTTPXMock):
         method="POST",
         match_headers={
             "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
-            "Authorization": "AWS4-HMAC-SHA256 Credential=access_id/20181011/us-east-1/iam/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=f3c8efd9b81b952035a73ea93d3a79380e13370bcaa6089e4275319bde17a400",
+            "Authorization": "AWS4-HMAC-SHA256 Credential=access_id/20181011/us-east-1/iam/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=04a5225313f4ffc8a8f4a974ad9c8d29a02df6ce0dabda1898ba1cccf2a3fb56",
             "x-amz-date": "20181011T150505Z",
         },
     )
@@ -824,7 +824,7 @@ async def test_aws_auth_path_percent_encode_non_s3(httpx_mock: HTTPXMock):
         method="POST",
         match_headers={
             "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
-            "Authorization": "AWS4-HMAC-SHA256 Credential=access_id/20181011/us-east-1/iam/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=7b3267f1b4bcb1f6731eb99aa9b3381225c18fc32e3ecb78fc4adceb746f92f3",
+            "Authorization": "AWS4-HMAC-SHA256 Credential=access_id/20181011/us-east-1/iam/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=9e643e5c1a494c954b28c0ad986b9343e70b02df2bdaddee7f7b2510073ae16c",
             "x-amz-date": "20181011T150505Z",
         },
     )

tests/aws_signature_v4/test_aws4auth_sync.py

@@ -774,7 +774,7 @@ def test_aws_auth_path_quoting(httpx_mock: HTTPXMock):
         method="POST",
         match_headers={
             "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
-            "Authorization": "AWS4-HMAC-SHA256 Credential=access_id/20181011/us-east-1/iam/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=f3c8efd9b81b952035a73ea93d3a79380e13370bcaa6089e4275319bde17a400",
+            "Authorization": "AWS4-HMAC-SHA256 Credential=access_id/20181011/us-east-1/iam/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=04a5225313f4ffc8a8f4a974ad9c8d29a02df6ce0dabda1898ba1cccf2a3fb56",
             "x-amz-date": "20181011T150505Z",
         },
     )
@@ -797,7 +797,7 @@ def test_aws_auth_path_percent_encode_non_s3(httpx_mock: HTTPXMock):
         method="POST",
         match_headers={
             "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
-            "Authorization": "AWS4-HMAC-SHA256 Credential=access_id/20181011/us-east-1/iam/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=7b3267f1b4bcb1f6731eb99aa9b3381225c18fc32e3ecb78fc4adceb746f92f3",
+            "Authorization": "AWS4-HMAC-SHA256 Credential=access_id/20181011/us-east-1/iam/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=9e643e5c1a494c954b28c0ad986b9343e70b02df2bdaddee7f7b2510073ae16c",
             "x-amz-date": "20181011T150505Z",
         },
     )