Decode non-ASCII request paths and WSGI query strings

src/hypercorn/app_wrappers.py

@@ -135,7 +135,7 @@ def _build_environ(scope: HTTPScope, body: bytes) -> dict:
         "REQUEST_METHOD": scope["method"],
         "SCRIPT_NAME": script_name.encode("utf8").decode("latin1"),
         "PATH_INFO": path.encode("utf8").decode("latin1"),
-        "QUERY_STRING": scope["query_string"].decode("ascii"),
+        "QUERY_STRING": scope["query_string"].decode("latin-1"),
         "SERVER_NAME": server[0],
         "SERVER_PORT": server[1],
         "SERVER_PROTOCOL": "HTTP/%s" % scope["http_version"],

src/hypercorn/protocol/events.py

@@ -1,10 +1,25 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
+from urllib.parse import unquote
 
 from hypercorn.typing import ConnectionState
 
 
+def decode_path(raw: bytes) -> str:
+    """Decode a request path for an ASGI scope.
+
+    Percent-encoded UTF-8 is ASCII until ``unquote``. Raw UTF-8 decodes as
+    UTF-8. Anything else falls back to latin-1 so one high byte does not
+    raise out of the protocol parser.
+    """
+    try:
+        text = raw.decode("utf-8")
+    except UnicodeDecodeError:
+        text = raw.decode("latin-1")
+    return unquote(text)
+
+
 @dataclass(frozen=True)
 class Event:
     stream_id: int

src/hypercorn/protocol/http_stream.py

@@ -3,10 +3,10 @@
 from collections.abc import Awaitable, Callable
 from enum import auto, Enum
 from time import time
-from urllib.parse import unquote
 
 from .events import (
     Body,
+    decode_path,
     EndBody,
     Event,
     InformationalResponse,
@@ -90,7 +90,7 @@ async def handle(self, event: Event) -> None:
                 "asgi": {"spec_version": "2.1", "version": "3.0"},
                 "method": event.method,
                 "scheme": self.scheme,
-                "path": unquote(path.decode("ascii")),
+                "path": decode_path(path),
                 "raw_path": path,
                 "query_string": query_string,
                 "root_path": self.config.root_path,

src/hypercorn/protocol/ws_stream.py

@@ -4,7 +4,6 @@
 from enum import auto, Enum
 from io import BytesIO, StringIO
 from time import time
-from urllib.parse import unquote
 
 from wsproto.connection import Connection, ConnectionState, ConnectionType
 from wsproto.events import (
@@ -20,7 +19,7 @@
 from wsproto.handshake import server_extensions_handshake, WEBSOCKET_VERSION
 from wsproto.utilities import generate_accept_token, LocalProtocolError, split_comma_header
 
-from .events import Body, Data, EndBody, EndData, Event, Request, Response, StreamClosed
+from .events import Body, Data, decode_path, EndBody, EndData, Event, Request, Response, StreamClosed
 from ..config import Config
 from ..typing import (
     AppWrapper,
@@ -212,7 +211,7 @@ async def handle(self, event: Event) -> None:
                 "asgi": {"spec_version": "2.3", "version": "3.0"},
                 "scheme": self.scheme,
                 "http_version": event.http_version,
-                "path": unquote(path.decode("ascii")),
+                "path": decode_path(path),
                 "raw_path": path,
                 "query_string": query_string,
                 "root_path": self.config.root_path,

tests/protocol/test_http_stream.py

@@ -72,6 +72,41 @@ async def test_handle_request_http_1(stream: HTTPStream, http_version: str) -> N
     }
 
 
+@pytest.mark.asyncio
+async def test_handle_request_non_ascii_path(stream: HTTPStream) -> None:
+    raw_path = "/café".encode()
+    await stream.handle(
+        Request(
+            stream_id=1,
+            http_version="1.1",
+            headers=[],
+            raw_path=raw_path,
+            method="GET",
+            state=ConnectionState({}),
+        )
+    )
+    scope = stream.task_group.spawn_app.call_args[0][2]  # type: ignore
+    assert scope["path"] == "/café"
+    assert scope["raw_path"] == raw_path
+
+
+@pytest.mark.asyncio
+async def test_handle_request_percent_encoded_path(stream: HTTPStream) -> None:
+    await stream.handle(
+        Request(
+            stream_id=1,
+            http_version="1.1",
+            headers=[],
+            raw_path=b"/%E4%B8%AD",
+            method="GET",
+            state=ConnectionState({}),
+        )
+    )
+    scope = stream.task_group.spawn_app.call_args[0][2]  # type: ignore
+    assert scope["path"] == "/中"
+    assert scope["raw_path"] == b"/%E4%B8%AD"
+
+
 @pytest.mark.asyncio
 async def test_handle_request_http_2(stream: HTTPStream) -> None:
     await stream.handle(

tests/test_app_wrappers.py

@@ -187,6 +187,27 @@ def test_build_environ_encoding() -> None:
     assert environ["PATH_INFO"] == "/文".encode().decode("latin-1")
 
 
+def test_build_environ_non_ascii_query_string() -> None:
+    scope: HTTPScope = {
+        "http_version": "1.1",
+        "asgi": {},
+        "method": "GET",
+        "headers": [],
+        "path": "/",
+        "root_path": "",
+        "query_string": "q=é".encode(),
+        "raw_path": b"/",
+        "scheme": "http",
+        "type": "http",
+        "client": None,
+        "server": None,
+        "extensions": {},
+        "state": ConnectionState({}),
+    }
+    environ = _build_environ(scope, b"")
+    assert environ["QUERY_STRING"].encode("latin-1") == scope["query_string"]
+
+
 def test_build_environ_root_path() -> None:
     scope: HTTPScope = {
         "http_version": "1.0",