From c084a6940b711783c477cc08bae541281b77d0e2 Mon Sep 17 00:00:00 2001 From: "carpentry-heartbeat[bot]" Date: Sat, 3 Oct 2026 11:04:56 +0200 Subject: [PATCH] =?UTF-8?q?Frame=20Content-Length=20bodies=20per=20RFC=209?= =?UTF-8?q?112=20=C2=A76.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Content-Length body now ends at its last byte instead of at the close: poll stops once the declared bytes are out, and anything after them is dropped, including bytes that arrived with the headers. The value must be 1*DIGIT and fit an Int. A comma list or repeated lines count as one value when every member agrees; a sign, junk, an empty value, differing values or a value too large to count fail with ClientError.Parse instead of wrapping or switching the truncation check off. Any Transfer-Encoding overrides Content-Length. --- README.md | 6 ++ http-client.carp | 150 +++++++++++++++++++++++++++++------------- test/http-client.carp | 145 ++++++++++++++++++++++++++++++++++++++++ test/server.py | 62 +++++++++++++++++ 4 files changed, 318 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 82be056..2d44ff7 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,12 @@ chunked body, or a `Content-Length` body that ends short, comes back as only by the connection closing declares no length to check against, so it is taken as complete however the connection ends. +A `Content-Length` body ends at its last byte: the stream does not wait for the +connection to close, and drops anything the server sends after it. A +`Content-Length` that is not a plain number, disagrees with another one, or is +too large for an `Int` fails the request with `ClientError.Parse`. Any +`Transfer-Encoding` overrides `Content-Length`. + ### Cookie jar Use a `CookieJar` to store cookies from responses and replay them on diff --git a/http-client.carp b/http-client.carp index 2eb86ee..4fcf525 100644 --- a/http-client.carp +++ b/http-client.carp @@ -309,12 +309,16 @@ apart. (do (when (= eol p) (set! scanning false)) (set! p (+ eol 2)))))) (set-pos! s p))) - (hidden charge!) - (private charge!) - (defn charge! [s n] - (match-ref (remaining s) - (Maybe.Nothing) () - (Maybe.Just left) (set-remaining! s (Maybe.Just (- @left n))))) + (hidden take!) + (private take!) + ; drops whatever lies past the declared length + (defn take! [s chunk] + (match @(remaining s) + (Maybe.Nothing) chunk + (Maybe.Just left) + (let-do [n (min left (String.length &chunk))] + (set-remaining! s (Maybe.Just (- left n))) + (String.byte-slice &chunk 0 n)))) (hidden truncated?) (private truncated?) @@ -332,17 +336,18 @@ apart. (hidden poll-raw) (private poll-raw) (defn poll-raw [s] - (if (> (String.length (buf s)) @(pos s)) - (let-do [leftover (unconsumed s)] - (set-pos! s (String.length (buf s))) - (charge! s (String.length &leftover)) - (Maybe.Just leftover)) + (cond + (= (remaining s) &(Maybe.Just 0)) (do (set-done! s true) (Maybe.Nothing)) + (> (String.length (buf s)) @(pos s)) + (let-do [leftover (unconsumed s)] + (set-pos! s (String.length (buf s))) + (Maybe.Just (take! s leftover))) (match (Connection.read (conn s)) (Result.Success chunk) (if (String.empty? &chunk) (end-raw! s @"truncated body: the connection closed before Content-Length bytes arrived") - (do (charge! s (String.length &chunk)) (Maybe.Just chunk))) + (Maybe.Just (take! s chunk))) (Result.Error e) (end-raw! s (fmt "read error: %s" &e))))) (hidden poll-chunked) @@ -569,15 +574,94 @@ to follow. Used by `request`, `request-stream`, and convenience methods.") (defn bodyless? [verb code] (or (= verb "HEAD") (or (< code 200) (or (= code 204) (= code 304))))) + (hidden parse-length) + (private parse-length) + ; RFC 9110 §8.6: 1*DIGIT, which must also fit an Int + (defn parse-length [s] + (let-do [len (String.length s) + acc 0 + digits (> len 0) + fits true] + (for [i 0 len] + (let [c (String.char-at s i) + d (- (Char.to-int c) 48)] + (cond + (not (Char.num? c)) (do (set! digits false) (break)) + (> acc (/ (- Int.MAX d) 10)) (set! fits false) + (set! acc (+ (* acc 10) d))))) + (cond + (not digits) + (Result.Error + (ClientError.Parse (fmt "invalid Content-Length '%s'" s))) + (not fits) + (Result.Error + (ClientError.Parse (fmt "Content-Length too large: %s" s))) + (Result.Success acc)))) + + (hidden agree-length) + (private agree-length) + ; RFC 9112 §6.3: repeated lengths must all be the same value + (defn agree-length [agreed member] + (match agreed + (Result.Error e) (Result.Error e) + (Result.Success prev) + (match (parse-length &(String.trim member)) + (Result.Error e) (Result.Error e) + (Result.Success n) + (match prev + (Maybe.Nothing) (Result.Success (Maybe.Just n)) + (Maybe.Just p) + (if (= p n) + (Result.Success (Maybe.Just n)) + (Result.Error + (ClientError.Parse + (fmt "conflicting Content-Length values %d and %d" p n)))))))) + + (hidden content-lengths) + (private content-lengths) + ; the members of every Content-Length field line, with comma lists split + (defn content-lengths [resp] + (Map.kv-reduce + &(fn [acc k vs] + (if (= &(String.ascii-to-lower k) "content-length") + (Array.reduce + &(fn [a v] (Array.concat &[a (String.split-by v &[\,])])) + acc + vs) + acc)) + (the (Array String) []) + (Response.headers resp))) + (hidden declared-length) (private declared-length) ; how many body bytes the response promises, or Nothing when it promises none (defn declared-length [resp verb code] - (if (bodyless? verb code) - (Maybe.Nothing) - (match (Response.header resp "Content-Length") - (Maybe.Nothing) (Maybe.Nothing) - (Maybe.Just v) (Int.from-string &v)))) + (if (or (bodyless? verb code) + (Maybe.just? &(Response.header resp "Transfer-Encoding"))) + (Result.Success (Maybe.Nothing)) + (Array.reduce &agree-length + (Result.Success (Maybe.Nothing)) + &(content-lengths resp)))) + + (hidden open-stream) + (private open-stream) + (defn open-stream [conn resp leftover verb] + (let [code @(Response.code &resp)] + (match (declared-length &resp verb code) + (Result.Error e) (do (Connection.close conn) (Result.Error e)) + (Result.Success left) + (let [is-chunked (and (Response.chunked? &resp) + (not (bodyless? verb code)))] + (Result.Success + (ResponseStream.init conn + leftover + 0 + (Maybe.Nothing) + is-chunked + false + left + code + resp)))))) ; RFC 9110 §15.4.2–§15.4.4 method rewriting. (hidden redirect-verb) @@ -720,21 +804,9 @@ to follow. Used by `request`, `request-stream`, and convenience methods.") (ClientError.Redirect (fmt "too many redirects (max %d)" max-redir)))) (break))) - (let-do [leftover @(Pair.b &pair) - is-chunked (and (Response.chunked? &resp) - (not (bodyless? &cur-verb code))) - left (declared-length &resp &cur-verb code)] + (do (set! result - (Result.Success - (ResponseStream.init conn - leftover - 0 - (Maybe.Nothing) - is-chunked - false - left - code - resp))) + (open-stream conn resp @(Pair.b &pair) &cur-verb)) (break))))))) result)) @@ -1049,21 +1121,9 @@ See `RequestConfig` for timeout and redirect details.") (ClientError.Redirect (fmt "too many redirects (max %d)" max-redir)))) (break))) - (let-do [leftover @(Pair.b &pair) - is-chunked (and (Response.chunked? &resp) - (not (bodyless? &cur-verb code))) - left (declared-length &resp &cur-verb code)] + (do (set! result - (Result.Success - (ResponseStream.init conn - leftover - 0 - (Maybe.Nothing) - is-chunked - false - left - code - resp))) + (open-stream conn resp @(Pair.b &pair) &cur-verb)) (break)))))))) result)) diff --git a/test/http-client.carp b/test/http-client.carp index 844e195..5f51595 100644 --- a/test/http-client.carp +++ b/test/http-client.carp @@ -108,6 +108,31 @@ (ResponseStream.close stream) kind)))) +; The body a stream yields poll by poll, then how it ended. +(defn streamed [url] + (match (Client.request-stream "GET" + url + (the (Map String (Array String)) {}) + "") + (Result.Error e) (ClientError.message &e) + (Result.Success stream) + (let-do [body @""] + (while-do true + (match (ResponseStream.poll &stream) + (Maybe.Nothing) (break) + (Maybe.Just chunk) (set! body (String.append &body &chunk)))) + (let-do [ended (match @(ResponseStream.error &stream) + (Maybe.Just e) (ClientError.message &e) + (Maybe.Nothing) @"clean")] + (ResponseStream.close stream) + (fmt "[%s] %s" &body &ended))))) + +; The held routes keep the connection open for 10 s after the body. +(defn without-waiting [f] + (let-do [t0 (System.time) + res (~f)] + (if (< (- (System.time) t0) 5) res @"waited for the close"))) + ; Returns the empty string on request failure. (defn get-body [url] (match (Client.get url) (Result.Success r) @(Response.body &r) _ @"")) @@ -954,6 +979,126 @@ &jar))) "the same holds on the cookie-jar path") + (assert-equal test + "200 [hello]" + &(status-and-body (Client.get "http://127.0.0.1:8791/length-extra")) + "bytes past Content-Length are not part of the body") + + (assert-equal test + "200 [hello]" + &(status-and-body (Client.get "http://127.0.0.1:8791/length-extra-late")) + "bytes past Content-Length in a later read are not part of the body") + + (assert-equal test + "[hello] clean" + &(streamed "http://127.0.0.1:8791/length-extra") + "a stream stops at Content-Length") + + (assert-equal test + "200 [hello]" + &(without-waiting + &(fn [] (status-and-body (Client.get "http://127.0.0.1:8791/length-held")))) + "a complete Content-Length body does not wait for the close") + + (assert-equal test + "[hello] clean" + &(without-waiting &(fn [] (streamed "http://127.0.0.1:8791/length-held"))) + "a stream ends at the last Content-Length byte without waiting for the close") + + (assert-equal test + "200 []" + &(without-waiting + &(fn [] + (status-and-body (Client.get "http://127.0.0.1:8791/length-zero-held")))) + "a zero Content-Length does not wait for the close") + + (assert-equal test + "invalid Content-Length '-5'" + &(status-and-body (Client.get "http://127.0.0.1:8791/length-negative")) + "a negative Content-Length is rejected") + + (assert-equal test + "invalid Content-Length ''" + &(status-and-body (Client.get "http://127.0.0.1:8791/length-empty")) + "an empty Content-Length is rejected") + + (assert-equal test + "invalid Content-Length '5x'" + &(status-and-body (Client.get "http://127.0.0.1:8791/length-junk")) + "a Content-Length with trailing junk is rejected") + + (assert-equal test + "invalid Content-Length '+5'" + &(status-and-body (Client.get "http://127.0.0.1:8791/length-plus")) + "a signed Content-Length is rejected") + + (assert-equal test + "parse" + &(error-kind (Client.get "http://127.0.0.1:8791/length-junk")) + "an invalid Content-Length is a Parse error") + + (assert-equal test + "200 [hello]" + &(status-and-body (Client.get "http://127.0.0.1:8791/length-list")) + "a list of identical Content-Length values counts as one") + + (assert-equal test + "200 [hello]" + &(status-and-body (Client.get "http://127.0.0.1:8791/length-lines")) + "repeated identical Content-Length lines count as one") + + (assert-equal test + "conflicting Content-Length values 5 and 6" + &(status-and-body (Client.get "http://127.0.0.1:8791/length-list-conflict")) + "a list of differing Content-Length values is rejected") + + (assert-equal test + "conflicting Content-Length values 3 and 5" + &(status-and-body (Client.get "http://127.0.0.1:8791/length-lines-conflict")) + "repeated differing Content-Length lines are rejected") + + (assert-equal test + "conflicting Content-Length values 3 and 5" + &(let-do [jar (CookieJar.create)] + (status-and-body + (Client.get-with-jar "http://127.0.0.1:8791/length-lines-conflict" &jar))) + "the same holds on the cookie-jar path") + + (assert-equal test + "truncated body: the connection closed before Content-Length bytes arrived" + &(status-and-body (Client.get "http://127.0.0.1:8791/length-max")) + "the largest Content-Length an Int holds is still a length") + + (assert-equal test + "Content-Length too large: 2147483648" + &(status-and-body (Client.get "http://127.0.0.1:8791/length-over-max")) + "a Content-Length one past the largest Int is rejected") + + (assert-equal test + "Content-Length too large: 4294967301" + &(status-and-body (Client.get "http://127.0.0.1:8791/length-wraps")) + "a Content-Length that would wrap to a small Int is rejected") + + (assert-equal test + "200 [hello]" + &(status-and-body (Client.get "http://127.0.0.1:8791/chunked-bad-length")) + "a chunked body ignores an invalid Content-Length") + + (assert-equal test + "200 [helloEXTRA]" + &(status-and-body (Client.get "http://127.0.0.1:8791/coded-length")) + "any Transfer-Encoding overrides Content-Length") + + (assert-equal test + "204 []" + &(status-and-body (Client.get "http://127.0.0.1:8791/no-content-bad-length")) + "a 204 ignores an invalid Content-Length") + + (assert-equal test + "200 []" + &(head-status-and-body "http://127.0.0.1:8791/length-negative") + "a HEAD response ignores an invalid Content-Length") + (assert-equal test "127.0.0.1:8791" &(seen-hosts (Client.get "http://127.0.0.1:8791/headers")) diff --git a/test/server.py b/test/server.py index 639e29f..5c3013b 100755 --- a/test/server.py +++ b/test/server.py @@ -62,6 +62,17 @@ def _chunked_raw(self, body): if self.command != "HEAD": self.wfile.write(body) + def _framed(self, fields, body, hold=0): + """A 200 with raw framing fields and body, kept open for `hold` seconds.""" + self.wfile.write( + b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n" + + fields + + b"Connection: close\r\n\r\n" + ) + if self.command != "HEAD": + self.wfile.write(body) + time.sleep(hold) + def _length_body(self, declared, sent): return ( b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: " @@ -239,6 +250,57 @@ def _route(self): b"5\r\nhello\r\n0\r\nX-Checksum: abc\r\nX-More: 1\r\n\r\n" ) + # Content-Length framing, each written as raw bytes + if path == "/length-extra": + return self._framed(b"Content-Length: 5\r\n", b"helloEXTRA") + if path == "/length-extra-late": + self._framed(b"Content-Length: 5\r\n", b"hel", hold=0.2) + self.wfile.write(b"loEXTRA") + return + if path == "/length-held": + return self._framed(b"Content-Length: 5\r\n", b"hello", hold=10) + if path == "/length-zero-held": + return self._framed(b"Content-Length: 0\r\n", b"", hold=10) + if path == "/length-negative": + return self._framed(b"Content-Length: -5\r\n", b"hello") + if path == "/length-empty": + return self._framed(b"Content-Length: \r\n", b"hello") + if path == "/length-junk": + return self._framed(b"Content-Length: 5x\r\n", b"hel") + if path == "/length-plus": + return self._framed(b"Content-Length: +5\r\n", b"hel") + if path == "/length-list": + return self._framed(b"Content-Length: 5, 5\r\n", b"helloEXTRA") + if path == "/length-list-conflict": + return self._framed(b"Content-Length: 5, 6\r\n", b"hello") + if path == "/length-lines": + return self._framed( + b"Content-Length: 5\r\ncontent-length: 5\r\n", b"helloEXTRA" + ) + if path == "/length-lines-conflict": + return self._framed(b"Content-Length: 3\r\nContent-Length: 5\r\n", b"hello") + if path == "/length-max": + return self._framed(b"Content-Length: 2147483647\r\n", b"hello") + if path == "/length-over-max": + return self._framed(b"Content-Length: 2147483648\r\n", b"hello") + if path == "/length-wraps": + return self._framed(b"Content-Length: 4294967301\r\n", b"hello") + if path == "/chunked-bad-length": + return self._framed( + b"Transfer-Encoding: chunked\r\nContent-Length: abc\r\n", + b"5\r\nhello\r\n0\r\n\r\n", + ) + if path == "/coded-length": + return self._framed( + b"Transfer-Encoding: xchunked\r\nContent-Length: 5\r\n", b"helloEXTRA" + ) + if path == "/no-content-bad-length": + self.wfile.write( + b"HTTP/1.1 204 No Content\r\nContent-Length: abc\r\n" + b"Connection: close\r\n\r\n" + ) + return + # /not-chunked: `chunked` as a substring of another coding token, with a # plain Content-Length body. if path == "/not-chunked":