diff --git a/README.md b/README.md index 6f4a9f6..fbc1f90 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,8 @@ For chunked or long-running responses, use `Client.request-stream` to get a (match (ResponseStream.poll &stream) (Maybe.Nothing) (break) (Maybe.Just chunk) (IO.print &chunk))) + (when (Maybe.just? (ResponseStream.error &stream)) + (IO.errorln "the response body was truncated")) (ResponseStream.close stream)) (Result.Error e) (IO.errorln &e)) ``` @@ -82,6 +84,15 @@ For chunked or long-running responses, use `Client.request-stream` to get a implements the `poll` interface from the [streams](https://github.com/carpentry-org/streams) library. +`poll` returns `Nothing` at the end of a well-formed body and also when the +framing or the transport failed part-way through it; `ResponseStream.error` +tells the two apart. The buffered request functions (`Client.get`, +`Client.request`, and friends) fold that into their result, so a malformed +chunked body, or a `Content-Length` body that ends short, comes back as +`(Result.Error …)` rather than a short body with a 200 on it. A body delimited +only by the connection closing declares no length to check against, so it is +taken as complete however the connection ends. + ### Cookie jar Use a `CookieJar` to store cookies from responses and replay them on diff --git a/http-client.carp b/http-client.carp index 21fd130..ea07eaa 100644 --- a/http-client.carp +++ b/http-client.carp @@ -22,8 +22,6 @@ (load "src/multipart.carp") (load "src/cookie-jar.carp") -(relative-include "src/chunked.h") - ; The `poll` interface for pull-based streams. Inlined here so that ; `ResponseStream` can implement it without an external dependency. If a ; `streams` package is published later, this declaration is compatible. @@ -89,16 +87,14 @@ currently support a timeout parameter.") "creates a RequestConfig with no timeouts and up to 10 redirects.") (defn default [] (RequestConfig.init 0 0 10))) -(private chunked-decode-one-) -(hidden chunked-decode-one-) -(register chunked-decode-one- - (Fn [(Ptr CChar) Int (Ptr String) (Ptr Int)] Int) - "chunked_decode_one") - (doc ResponseStream "is a pull-based stream for reading HTTP response bodies. Handles chunked transfer-encoding automatically. Implements the `poll` interface from the streams library. +`poll` returns `Nothing` both at the end of a well-formed body and when the +framing or the transport failed part-way through it; `error` tells the two +apart. + ``` (match (Client.request-stream \"POST\" url headers body) (Result.Success stream) @@ -107,80 +103,192 @@ from the streams library. (match (ResponseStream.poll &stream) (Maybe.Nothing) (break) (Maybe.Just chunk) (IO.print &chunk))) + (when (Maybe.just? (ResponseStream.error &stream)) + (IO.errorln \"the response body was truncated\")) (ResponseStream.close stream)) (Result.Error e) (IO.errorln &e)) ```") (deftype ResponseStream [conn Connection buf String - decoded String + pos Int + error (Maybe String) chunked Bool done Bool + remaining (Maybe Int) status-code Int parsed-response Response]) (defmodule ResponseStream + (doc error "returns the framing or transport error that ended the stream, or +`Nothing` if it ended cleanly.") + + (hidden hex-nibble) + (private hex-nibble) + (defn hex-nibble [c] + (cond + (and (>= c \0) (<= c \9)) (- (Char.to-int c) 48) + (and (>= c \a) (<= c \f)) (+ 10 (- (Char.to-int c) 97)) + (and (>= c \A) (<= c \F)) (+ 10 (- (Char.to-int c) 65)) + -1)) + + (hidden parse-hex) + (private parse-hex) + (defn parse-hex [s] + (let-do [len (String.length s) + acc 0 + ok (> len 0)] + (for [i 0 len] + (let [d (hex-nibble (String.char-at s i))] + (cond + (< d 0) + (do (set! ok false) (break)) + ; reject a size that would overflow a signed 32-bit Int (acc*16+d) + (> acc (/ (- Int.MAX d) 16)) + (do (set! ok false) (break)) + (set! acc (+ (* acc 16) d))))) + (if ok (Maybe.Just acc) (Maybe.Nothing)))) + + (hidden crlf-at?) + (private crlf-at?) + (defn crlf-at? [s i] + (and (= (String.char-at s i) \return) + (= (String.char-at s (Int.inc i)) \newline))) + + (hidden fail!) + (private fail!) + (defn fail! [s msg] (do (set-error! s (Maybe.Just msg)) (set-done! s true))) + + (hidden unconsumed) + (private unconsumed) + (defn unconsumed [s] + (String.byte-slice (buf s) @(pos s) (String.length (buf s)))) + + (hidden fill!) + (private fill!) + ; appends one read to the unconsumed tail; false at EOF or on a transport error + (defn fill! [s] + (match (Connection.read (conn s)) + (Result.Error e) (do (fail! s (fmt "read error: %s" &e)) false) + (Result.Success chunk) + (if (String.empty? &chunk) + false + (let-do [grown (String.append &(unconsumed s) &chunk)] + (set-buf! s grown) + (set-pos! s 0) + true)))) + + (hidden skip-trailers!) + (private skip-trailers!) + ; consumes as much of the trailer section as is already buffered + (defn skip-trailers! [s] + (let-do [len (String.length (buf s)) + p @(pos s) + scanning true] + (while scanning + (let [eol (String.find-crlf (buf s) p len)] + (if (= eol -1) + (set! scanning false) + (do (when (= eol p) (set! scanning false)) (set! p (+ eol 2)))))) + (set-pos! s p))) + + (hidden charge!) + (private charge!) + (defn charge! [s n] + (match-ref (remaining s) + (Maybe.Nothing) () + (Maybe.Just left) (set-remaining! s (Maybe.Just (- @left n))))) + + (hidden truncated?) + (private truncated?) + ; a body with no declared length ends at EOF, so it can never be short + (defn truncated? [s] + (match-ref (remaining s) (Maybe.Nothing) false (Maybe.Just left) (> @left 0))) + + (hidden end-raw!) + (private end-raw!) + (defn end-raw! [s why] + (do (if (truncated? s) (fail! s why) (set-done! s true)) (Maybe.Nothing))) + (hidden poll-raw) (private poll-raw) (defn poll-raw [s] - ; Return any leftover bytes from the header read first - (if (> (String.length (buf s)) 0) - (let-do [leftover @(buf s)] (set-buf! s @"") (Maybe.Just leftover)) + (if (> (String.length (buf s)) @(pos s)) + (let-do [leftover (unconsumed s)] + (set-pos! s (String.length (buf s))) + (charge! s (String.length &leftover)) + (Maybe.Just leftover)) (match (Connection.read (conn s)) (Result.Success chunk) - (if (= (String.length &chunk) 0) - (do (set-done! s true) (Maybe.Nothing)) - (Maybe.Just chunk)) - (Result.Error _) (do (set-done! s true) (Maybe.Nothing))))) + (if (String.empty? &chunk) + (end-raw! s + @"truncated body: the connection closed before Content-Length bytes arrived") + (do (charge! s (String.length &chunk)) (Maybe.Just chunk))) + (Result.Error e) (end-raw! s (fmt "read error: %s" &e))))) (hidden poll-chunked) (private poll-chunked) (defn poll-chunked [s] - ; try to decode from existing buffer first, reading more if needed (let-do [result (Maybe.Nothing) - sb (StringBuf.create)] - (while-do (Maybe.nothing? &result) - (StringBuf.clear &sb) - (StringBuf.append-str &sb (buf s)) - (StringBuf.append-str &sb (decoded s)) - (let [raw (StringBuf.to-string &sb)] - (let-do [out @"" - consumed 0 - rc (chunked-decode-one- (String.cstr &raw) - (String.length &raw) - (Pointer.address &out) - (Pointer.address &consumed))] - (cond - (= rc 1) - ; got a chunk - (do - ; `consumed` is a byte count from the decoder, so the - ; remainder must be taken by byte offset. String.suffix counts - ; characters, which drops too much as soon as a chunk holds - ; multi-byte UTF-8 and desyncs us from the chunk framing. - (set-buf! s - (String.byte-slice &raw consumed (String.length &raw))) - (set-decoded! s @"") - (set! result (Maybe.Just out))) - (= rc 0) - ; need more data - (match (Connection.read (conn s)) - (Result.Success chunk) - (if (= (String.length &chunk) 0) - (do (set-done! s true) (break)) + running true] + (while running + (let [len (String.length (buf s)) + p @(pos s) + eol (String.find-crlf (buf s) p len)] + (if (= eol -1) + (unless-do (fill! s) + (when (Maybe.nothing? (error s)) + (fail! s + (if (< p len) + @"malformed chunked body: unterminated chunk size line" + @"malformed chunked body: missing terminating zero-size chunk"))) + (set! running false)) + (let [line &(String.byte-slice (buf s) p eol) + semi (String.index-of line \;) + hex &(String.trim + &(if (= semi -1) @line (String.byte-slice line 0 semi)))] + (match (parse-hex hex) + (Maybe.Nothing) + (do + (fail! s + (fmt + "malformed chunked body: invalid chunk size '%s'" + hex)) + (set! running false)) + (Maybe.Just size) + (if (= size 0) (do - (StringBuf.clear &sb) - (StringBuf.append-str &sb (buf s)) - (StringBuf.append-str &sb &chunk) - (set-buf! s (StringBuf.to-string &sb)))) - (Result.Error _) (do (set-done! s true) (break))) - ; rc < 0: end of stream (-1) or parse error (-2) - (do (set-done! s true) (set! result (Maybe.Nothing)) (break)))))) - (StringBuf.delete sb) + (set-pos! s (+ eol 2)) + (skip-trailers! s) + (set-done! s true) + (set! running false)) + (let [start (+ eol 2) + avail (- len start)] + ; compared without adding, so a huge size cannot overflow + (if (> size (- avail 2)) + (unless-do (fill! s) + (when (Maybe.nothing? (error s)) + (fail! s + @"malformed chunked body: truncated chunk data")) + (set! running false)) + (let-do [end (+ start size)] + (if (crlf-at? (buf s) end) + (do + (set! result + (Maybe.Just (String.byte-slice (buf s) + start + end))) + (set-pos! s (+ end 2))) + (fail! s + @"malformed chunked body: chunk data missing CRLF")) + (set! running false)))))))))) result)) - (doc poll "returns the next chunk of decoded body data, or `Nothing` when -the response is complete.") + (doc poll "returns the next chunk of decoded body data, or `Nothing` when the +response is complete. + +`Nothing` is also returned when the framing or the transport failed, so a caller +that cares about a truncated body must check `error` afterwards.") (defn poll [s] (cond @(done s) (Maybe.Nothing) @(chunked s) (poll-chunked s) (poll-raw s))) (implements poll ResponseStream.poll) @@ -316,6 +424,22 @@ to follow. Used by `request`, `request-stream`, and convenience methods.") (or (= code 301) (or (= code 302) (or (= code 303) (or (= code 307) (= code 308)))))) + (hidden bodyless?) + (private bodyless?) + ; RFC 9110 §6.4.1: these responses carry no body, whatever they frame it as. + (defn bodyless? [verb code] + (or (= verb "HEAD") (or (< code 200) (or (= code 204) (= code 304))))) + + (hidden declared-length) + (private declared-length) + ; how many body bytes the response promises, or Nothing when it promises none + (defn declared-length [resp verb code] + (if (bodyless? verb code) + (Maybe.Nothing) + (match (Response.header resp "Content-Length") + (Maybe.Nothing) (Maybe.Nothing) + (Maybe.Just v) (Int.from-string &v)))) + ; RFC 9110 §15.4.2–§15.4.4 method rewriting. (hidden redirect-verb) (private redirect-verb) @@ -474,14 +598,18 @@ to follow. Used by `request`, `request-stream`, and convenience methods.") (fmt "too many redirects (max %d)" max-redir))) (break))) (let-do [leftover @(Pair.b &pair) - is-chunked (Response.chunked? &resp)] + is-chunked (and (Response.chunked? &resp) + (not (bodyless? &cur-verb code))) + left (declared-length &resp &cur-verb code)] (set! result (Result.Success (ResponseStream.init conn leftover - @"" + 0 + (Maybe.Nothing) is-chunked false + left code resp))) (break))))))) @@ -535,23 +663,32 @@ See `RequestConfig` for timeout and redirect details.") (match (ResponseStream.poll s) (Maybe.Nothing) (set! done true) (Maybe.Just chunk) (StringBuf.append-str &sb &chunk))) - (let-do [body (StringBuf.to-string &sb)] (StringBuf.delete sb) body))) + (let-do [body (StringBuf.to-string &sb)] + (StringBuf.delete sb) + (match @(ResponseStream.error s) + (Maybe.Just e) (Result.Error e) + (Maybe.Nothing) (Result.Success body))))) + + (hidden collect-response) + (private collect-response) + (defn collect-response [streamed] + (match streamed + (Result.Error e) (Result.Error e) + (Result.Success stream) + (let-do [drained (drain-stream &stream) + base-resp @(ResponseStream.parsed-response &stream)] + (ResponseStream.close stream) + (match drained + (Result.Error e) (Result.Error e) + (Result.Success body) + (Result.Success (Response.set-body base-resp body)))))) (doc request-with-max-redirects "sends an HTTP request, following up to `max-redirects` redirects. Returns `(Result Response String)`. Pass 0 to disable redirect following.") (defn request-with-max-redirects [verb url headers body max-redirects] - (match (request-stream-with-max-redirects verb - url - headers - body - max-redirects) - (Result.Error e) (Result.Error e) - (Result.Success stream) - (let-do [decoded-body (drain-stream &stream) - base-resp @(ResponseStream.parsed-response &stream)] - (ResponseStream.close stream) - (Result.Success (Response.set-body base-resp decoded-body))))) + (collect-response + (request-stream-with-max-redirects verb url headers body max-redirects))) (doc request "sends an HTTP request to the given URL. Returns `(Result Response String)`. @@ -575,13 +712,7 @@ Returns `(Result Response String)`. See `RequestConfig` for timeout and redirect details.") (defn request-with-config [verb url headers body config] - (match (request-stream-with-config verb url headers body config) - (Result.Error e) (Result.Error e) - (Result.Success stream) - (let-do [decoded-body (drain-stream &stream) - base-resp @(ResponseStream.parsed-response &stream)] - (ResponseStream.close stream) - (Result.Success (Response.set-body base-resp decoded-body))))) + (collect-response (request-stream-with-config verb url headers body config))) (doc get "performs an HTTP GET request. Returns `(Result Response String)`.") (defn get [url] (request "GET" url (the (Map String (Array String)) {}) "")) @@ -775,14 +906,18 @@ See `RequestConfig` for timeout and redirect details.") (fmt "too many redirects (max %d)" max-redir))) (break))) (let-do [leftover @(Pair.b &pair) - is-chunked (Response.chunked? &resp)] + is-chunked (and (Response.chunked? &resp) + (not (bodyless? &cur-verb code))) + left (declared-length &resp &cur-verb code)] (set! result (Result.Success (ResponseStream.init conn leftover - @"" + 0 + (Maybe.Nothing) is-chunked false + left code resp))) (break)))))))) @@ -806,24 +941,13 @@ Matching cookies are sent automatically, and Set-Cookie response headers are stored in the jar. Returns `(Result Response String)`. Follows up to `default-max-redirects` redirects.") (defn request-with-jar [verb url headers body jar] - (match (request-stream-with-jar verb url headers body jar) - (Result.Error e) (Result.Error e) - (Result.Success stream) - (let-do [decoded-body (drain-stream &stream) - base-resp @(ResponseStream.parsed-response &stream)] - (ResponseStream.close stream) - (Result.Success (Response.set-body base-resp decoded-body))))) + (collect-response (request-stream-with-jar verb url headers body jar))) (doc request-with-jar-and-config "sends an HTTP request using the given `CookieJar` and `RequestConfig`. Returns `(Result Response String)`.") (defn request-with-jar-and-config [verb url headers body jar config] - (match (request-stream-with-jar-and-config verb url headers body jar config) - (Result.Error e) (Result.Error e) - (Result.Success stream) - (let-do [decoded-body (drain-stream &stream) - base-resp @(ResponseStream.parsed-response &stream)] - (ResponseStream.close stream) - (Result.Success (Response.set-body base-resp decoded-body))))) + (collect-response + (request-stream-with-jar-and-config verb url headers body jar config))) (doc get-with-jar "performs an HTTP GET request using the given `CookieJar`. Returns `(Result Response String)`.") diff --git a/src/chunked.h b/src/chunked.h deleted file mode 100644 index 05e683d..0000000 --- a/src/chunked.h +++ /dev/null @@ -1,71 +0,0 @@ -#ifndef CARP_CHUNKED_H -#define CARP_CHUNKED_H - -#include -#include -#include -#include - -/* Maximum chunk size: 16 MiB. Prevents malicious servers from triggering - * unbounded allocations via an enormous chunk-size line. */ -#define CHUNKED_MAX_CHUNK_SIZE (16 * 1024 * 1024) - -/* Decode one chunk from a chunked transfer-encoding buffer. - * - * Input: buf contains raw chunked data (e.g. "1a\r\n\r\n...") - * Output: decoded chunk data written to *out, bytes consumed written to *consumed. - * - * Returns: - * 1 = decoded a chunk (data in *out, length in *consumed) - * 0 = need more data (incomplete chunk in buffer) - * -1 = end of stream (chunk size 0) - * -2 = parse error (invalid hex, overflow, or chunk too large) - */ -int chunked_decode_one(const char *buf, int buf_len, String *out, int *consumed) { - /* Find \r\n to read chunk size */ - const char *crlf = NULL; - for (int i = 0; i < buf_len - 1; i++) { - if (buf[i] == '\r' && buf[i + 1] == '\n') { - crlf = buf + i; - break; - } - } - if (!crlf) return 0; /* need more data */ - - /* The chunk-size line must start with a hex digit (RFC 7230 §4.1). */ - if (buf == crlf || !isxdigit((unsigned char)buf[0])) return -2; - - /* Parse hex chunk size, validating that strtol consumed meaningful input - * and stopped at an expected delimiter (\r for end-of-size, or ; for a - * chunk extension per RFC 7230). */ - char *endptr = NULL; - long chunk_size = strtol(buf, &endptr, 16); - if (endptr == buf || (endptr != (char *)crlf && *endptr != ';')) return -2; - - /* Reject negative values (sign prefix) and sizes above the safety cap. */ - if (chunk_size < 0 || chunk_size > CHUNKED_MAX_CHUNK_SIZE) return -2; - - if (chunk_size == 0) { - *consumed = (int)(crlf - buf) + 2; /* skip "0\r\n" */ - *out = CARP_MALLOC(1); - (*out)[0] = '\0'; - return -1; /* end of stream */ - } - - int header_len = (int)(crlf - buf) + 2; /* "1a\r\n" */ - - /* Guard against int overflow in the total-needed calculation. */ - if (chunk_size > INT_MAX - header_len - 2) return -2; - int needed = header_len + (int)chunk_size + 2; /* +2 for trailing \r\n */ - - if (buf_len < needed) return 0; /* need more data */ - - /* Extract chunk data */ - *out = CARP_MALLOC(chunk_size + 1); - memcpy(*out, buf + header_len, chunk_size); - (*out)[chunk_size] = '\0'; - *consumed = needed; - return 1; -} - -#endif diff --git a/test/http-client.carp b/test/http-client.carp index 33f9cc4..da7cbb6 100644 --- a/test/http-client.carp +++ b/test/http-client.carp @@ -22,6 +22,26 @@ (ResponseStream.close stream) n))) +; The framing error the stream ended on, or how many chunks it ended cleanly +; after, so a clean end can never read as a failure. +(defn stream-error [url] + (match (Client.request-stream "GET" + url + (the (Map String (Array String)) {}) + "") + (Result.Error e) e + (Result.Success stream) + (let-do [n 0] + (while-do true + (match (ResponseStream.poll &stream) + (Maybe.Nothing) (break) + (Maybe.Just _) (set! n (Int.inc n)))) + (let-do [e (match @(ResponseStream.error &stream) + (Maybe.Just e) e + (Maybe.Nothing) (fmt "clean after %d chunks" n))] + (ResponseStream.close stream) + e)))) + ; Returns the empty string on request failure. (defn get-body [url] (match (Client.get url) (Result.Success r) @(Response.body &r) _ @"")) @@ -622,4 +642,108 @@ &(get-body "http://127.0.0.1:8791/header-continuation") "a header value that is not valid UTF-8 leaves the body intact") + (assert-equal test + "malformed chunked body: invalid chunk size 'zz'" + &(status-and-body (Client.get "http://127.0.0.1:8791/chunked-bad-hex")) + "a bad hex chunk size is an error, not a short body") + + (assert-equal test + "malformed chunked body: invalid chunk size '0x5'" + &(status-and-body (Client.get "http://127.0.0.1:8791/chunked-hex-prefix")) + "a 0x-prefixed chunk size is rejected") + + (assert-equal test + "malformed chunked body: truncated chunk data" + &(status-and-body (Client.get "http://127.0.0.1:8791/chunked-truncated")) + "a chunk cut off mid-data is an error, not a short body") + + (assert-equal test + "malformed chunked body: missing terminating zero-size chunk" + &(status-and-body (Client.get "http://127.0.0.1:8791/chunked-no-terminator")) + "a body that never sends the zero-size chunk is an error") + + (assert-equal test + "malformed chunked body: chunk data missing CRLF" + &(status-and-body (Client.get "http://127.0.0.1:8791/chunked-missing-crlf")) + "chunk data not followed by CRLF is an error") + + (assert-equal test + "malformed chunked body: truncated chunk data" + &(status-and-body (Client.get "http://127.0.0.1:8791/chunked-oversize")) + "a chunk size above the old 16 MiB cap is no longer a parse error") + + (assert-equal test + "malformed chunked body: invalid chunk size 'zz'" + &(let-do [jar (CookieJar.create)] + (status-and-body + (Client.get-with-jar "http://127.0.0.1:8791/chunked-bad-hex" &jar))) + "the same holds on the cookie-jar path") + + (assert-equal test + "200 [hello]" + &(status-and-body (Client.get "http://127.0.0.1:8791/chunked-ext")) + "a chunk extension after the size is ignored") + + (assert-equal test + "200 [hello]" + &(status-and-body (Client.get "http://127.0.0.1:8791/chunked-trailer")) + "a trailer section after the last chunk is consumed, not decoded") + + (assert-equal test + "200 []" + &(head-status-and-body "http://127.0.0.1:8791/chunked-trailer") + "a HEAD response announcing chunked has no body to decode") + + (assert-equal test + "clean after 40 chunks" + &(stream-error "http://127.0.0.1:8791/chunked-utf8/40") + "a well-formed chunked stream ends with no error") + + (assert-equal test + "clean after 1 chunks" + &(stream-error "http://127.0.0.1:8791/chunked-trailer") + "a trailered chunked stream ends with no error") + + (assert-equal test + "malformed chunked body: truncated chunk data" + &(stream-error "http://127.0.0.1:8791/chunked-truncated") + "a stream that dies mid-chunk carries the reason") + + (assert-equal test + "413 [payload too large]" + &(status-and-body + (Client.post "http://127.0.0.1:8791/reject-early" + {} + &(String.repeat 32768 "x"))) + "an early reject that resets the connection keeps its status and body") + + (assert-equal test + "200 [complete!!]" + &(status-and-body (Client.get "http://127.0.0.1:8791/reset-complete")) + "a complete Content-Length body outlives a reset after it") + + (assert-true test + (Result.error? &(Client.get "http://127.0.0.1:8791/reset-short")) + "a reset before Content-Length bytes arrive is an error") + + (assert-equal test + "truncated body: the connection closed before Content-Length bytes arrived" + &(status-and-body (Client.get "http://127.0.0.1:8791/close-short")) + "a clean close before Content-Length bytes arrive is an error") + + (assert-equal test + "200 [eof-delimited]" + &(status-and-body (Client.get "http://127.0.0.1:8791/reset-no-length")) + "a body with no declared length ends at the reset, not in an error") + + (assert-equal test + "413 [payload too large]" + &(let-do [jar (CookieJar.create)] + (status-and-body + (Client.post-with-jar "http://127.0.0.1:8791/reject-early" + {} + &(String.repeat 32768 "x") + &jar))) + "the same holds on the cookie-jar path") + (cookie-jar-tests test)) diff --git a/test/server.py b/test/server.py index afef0c4..5c70938 100755 --- a/test/server.py +++ b/test/server.py @@ -7,6 +7,8 @@ served on every port; the cross-origin test just points at a second port. """ +import socket +import struct import sys import time from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -49,6 +51,35 @@ def _raw(self, header_line, body): + body ) + def _chunked_raw(self, body): + """Chunked response whose body bytes are written exactly as given.""" + self.wfile.write( + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: text/plain\r\n" + b"Transfer-Encoding: chunked\r\n" + b"Connection: close\r\n\r\n" + ) + if self.command != "HEAD": + self.wfile.write(body) + + def _length_body(self, declared, sent): + return ( + b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: " + + str(declared).encode() + + b"\r\nConnection: close\r\n\r\n" + + sent + ) + + def _abort(self, raw): + """Writes raw bytes, then resets the connection instead of closing it.""" + self.wfile.write(raw) + time.sleep(0.25) # give the client time to read before the reset lands + self.connection.setsockopt( + socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0) + ) + self.connection.close() + self.close_connection = True + def _body_bytes(self): n = int(self.headers.get("Content-Length", 0) or 0) return self.rfile.read(n) if n else b"" @@ -166,6 +197,32 @@ def _route(self): self.wfile.write(b"%x\r\n%s\r\n0\r\n\r\n" % (len(body), body)) return + # chunked framing faults, each written as raw bytes so the exact wire + # form reaches the client + if path == "/chunked-bad-hex": + return self._chunked_raw(b"zz\r\nhello\r\n0\r\n\r\n") + if path == "/chunked-hex-prefix": + return self._chunked_raw(b"0x5\r\nhello\r\n0\r\n\r\n") + if path == "/chunked-truncated": + return self._chunked_raw(b"10\r\nshort") + if path == "/chunked-no-terminator": + return self._chunked_raw(b"5\r\nhello\r\n") + if path == "/chunked-missing-crlf": + return self._chunked_raw(b"5\r\nhelloXX0\r\n\r\n") + + # a chunk size above 16 MiB, whose data is never sent, so the size + # line alone decides the outcome + if path == "/chunked-oversize": + return self._chunked_raw(b"1000001\r\nhello") + + # a well-formed body with a chunk extension and a trailer section + if path == "/chunked-ext": + return self._chunked_raw(b"5;name=value\r\nhello\r\n0\r\n\r\n") + if path == "/chunked-trailer": + return self._chunked_raw( + b"5\r\nhello\r\n0\r\nX-Checksum: abc\r\nX-More: 1\r\n\r\n" + ) + # /not-chunked: `chunked` as a substring of another coding token, with a # plain Content-Length body. if path == "/not-chunked": @@ -173,6 +230,31 @@ def _route(self): 200, "plain-body", extra=[("Transfer-Encoding", "xchunked")] ) + # Content-Length bodies that end in a connection reset rather than a + # clean close, and one that ends short with a clean close. + if path == "/reset-complete": + return self._abort(self._length_body(10, b"complete!!")) + if path == "/reset-short": + return self._abort(self._length_body(64, b"only-ten-b")) + if path == "/close-short": + self.wfile.write(self._length_body(64, b"only-ten-b")) + self.close_connection = True + return + if path == "/reset-no-length": + return self._abort( + b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n" + b"Connection: close\r\n\r\neof-delimited" + ) + + # /reject-early: answers without draining the request body, then resets, + # which is what a 413, 401 or 400 on an upload looks like on the wire. + if path == "/reject-early": + return self._abort( + b"HTTP/1.1 413 Payload Too Large\r\nContent-Type: text/plain\r\n" + b"Content-Length: 17\r\nConnection: close\r\n\r\n" + b"payload too large" + ) + # /header-utf8 and /header-continuation: header values that are not # ASCII, written as raw bytes so the exact encoding reaches the client. if path == "/header-utf8":