From 2fd8e77919bf427c5fa7d49037a98d19d5403bac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Rubinos=20Rodr=C3=ADguez?= Date: Thu, 24 Sep 2026 12:09:36 +0200 Subject: [PATCH] feat(hpack): add an indexing policy to the encoder --- CHANGELOG.md | 12 ++ src/nhttp_h2.erl | 41 ++++- src/nhttp_hpack.erl | 116 +++++++++++++- test/nhttp_h2_SUITE.erl | 46 +++++- test/nhttp_hpack_SUITE.erl | 194 ++++++++++++++++++++++- test/nhttp_props_SUITE.erl | 6 +- test/property_test/nhttp_hpack_props.erl | 49 ++++++ 7 files changed, 444 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 644aba0..1efe5e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `nhttp_h2:stream_send_window/2` reads the send window of one stream - `nhttp_h2:peer_settings/1` reads the settings the peer sent, merged over the defaults +- `nhttp_hpack:new/2` takes an indexing policy of type + `t:nhttp_hpack:index_policy/0`. A field name in `no_index` encodes as a + literal without indexing (RFC 7541 Section 6.2.2) and a name in + `never_index` as a literal never indexed (RFC 7541 Section 6.2.3). Neither + enters the dynamic table, and both keep a static name reference. The + default of `new/0` and `new/1` is the empty policy, which changes no byte + of output +- `hpack_index_policy` on `t:nhttp_h2:settings/0` carries the policy to the + HPACK encoder of the connection. It has no wire representation +- `nhttp_hpack:table_entries/1`, `nhttp_h2:encoder_table_entries/1` and + `nhttp_h2:encoder_table_size/1` read the live entry count and the octets + of the encoder dynamic table ### Changed diff --git a/src/nhttp_h2.erl b/src/nhttp_h2.erl index 522411d..2d4945d 100644 --- a/src/nhttp_h2.erl +++ b/src/nhttp_h2.erl @@ -101,6 +101,8 @@ Flow-control and END_STREAM ride on the DATA frame. %%%----------------------------------------------------------------------------- -export([ connection_send_window/1, + encoder_table_entries/1, + encoder_table_size/1, peer_settings/1, stream_send_window/2 ]). @@ -139,9 +141,13 @@ Flow-control and END_STREAM ride on the DATA frame. Connection settings. Most keys map to a SETTINGS parameter of RFC 9113 Section 6.5.2 and go on -the wire. `max_continuation_frames` and `max_reset_streams` are local -policy. They bound work that a peer can request, they have no wire -representation, and the encoder drops them. +the wire. `max_continuation_frames`, `max_reset_streams` and +`hpack_index_policy` are local policy. They have no wire representation, +and the encoder drops them. The first two bound work that a peer can +request. `hpack_index_policy` reaches `nhttp_hpack:new/2` and names the +fields that the HPACK encoder keeps out of its dynamic table. The +`nhttp_hpack` module documentation gives the two literal forms and the +sets that fit a deployment. `max_continuation_frames` bounds the number of CONTINUATION frames in one field section, per RFC 9113 Section 10.5. An empty CONTINUATION frame adds @@ -161,7 +167,8 @@ CONTINUATION frames that carry the larger field section. max_header_list_size => pos_integer() | infinity, max_continuation_frames => pos_integer() | infinity, enable_connect_protocol => boolean(), - max_reset_streams => pos_integer() | infinity + max_reset_streams => pos_integer() | infinity, + hpack_index_policy => nhttp_hpack:index_policy() }. -type stream_state() :: @@ -298,7 +305,8 @@ new(Role) -> new(Role, LocalSettings) -> MergedSettings = maps:merge(default_settings(), LocalSettings), HeaderTableSize = maps:get(header_table_size, MergedSettings, ?H2_DEFAULT_HEADER_TABLE_SIZE), - {ok, HpackEnc} = nhttp_hpack:new(HeaderTableSize), + IndexPolicy = maps:get(hpack_index_policy, MergedSettings, #{}), + {ok, HpackEnc} = nhttp_hpack:new(HeaderTableSize, IndexPolicy), {ok, HpackDec} = nhttp_hpack:new(HeaderTableSize), #h2_conn{ role = Role, @@ -577,6 +585,29 @@ takes it positive again. connection_send_window(#h2_conn{send_window = Window}) -> Window. +-doc """ +Return the number of live entries in the dynamic table of the HPACK encoder. + +The count moves with every field that `send_headers/4` inserts and with +every eviction that the insert forces (RFC 7541 Section 4.4). A field whose +name is in `hpack_index_policy` never inserts, so a caller that tunes the +policy reads its effect here. +""". +-spec encoder_table_entries(conn()) -> non_neg_integer(). +encoder_table_entries(#h2_conn{hpack_enc = HpackEnc}) -> + nhttp_hpack:table_entries(HpackEnc). + +-doc """ +Return the size of the dynamic table of the HPACK encoder in octets. + +The size counts every live entry as name, value and the 32 octets of +overhead of RFC 7541 Section 4.1. It never exceeds the +SETTINGS_HEADER_TABLE_SIZE that the peer sent. +""". +-spec encoder_table_size(conn()) -> non_neg_integer(). +encoder_table_size(#h2_conn{hpack_enc = HpackEnc}) -> + nhttp_hpack:table_size(HpackEnc). + -doc """ Return the settings the peer sent, merged over the defaults. diff --git a/src/nhttp_hpack.erl b/src/nhttp_hpack.erl index ab2de0d..c4404c8 100644 --- a/src/nhttp_hpack.erl +++ b/src/nhttp_hpack.erl @@ -52,6 +52,40 @@ field that arrives by index carries that verdict out again and needs no second read of the octets. The verdict names the part that failed, so a field that reuses the name by index drops a verdict against the value and reads a fresh value from the wire. + +## Indexing policy + +`new/2` takes a policy that names the fields the encoder keeps out of the +dynamic table. RFC 7541 §6.2 gives three literal representations, and the +policy selects one for each field name: + +- A name in neither list uses literal with incremental indexing (§6.2.1). + The field enters the dynamic table of both endpoints. +- A name in `no_index` uses literal without indexing (§6.2.2). The field + enters no table on this hop. Use it for a value that differs on every + request. Such a value costs a table entry that no later request reuses. +- A name in `never_index` uses literal never indexed (§6.2.3). Every + intermediary keeps this form on every later hop. Use it for a secret: + RFC 7541 §7.1 shows how an attacker who places values in the same block + and reads the compressed size recovers an indexed secret, and §7.1.3 + names this form as the mitigation. + +A name in both lists resolves to `never_index`. The policy holds every name +in lowercase. A name with a static entry keeps its static name reference +under both literal forms. The default of `new/0` and `new/1` is the empty +policy, which changes no byte of output. + +The dynamic table evicts in insertion order (§4.4). One field that differs +on every request and still inserts pushes every constant entry out once +the table fills, and the encoder then re-emits the constant fields as +literals. A `no_index` list must therefore name every such field. + +A client that shares a connection between principals, or that compresses +a value an attacker controls in the same block as a credential, puts +`authorization`, `cookie` and `proxy-authorization` in `never_index`. A +client with one principal on a TLS connection and no attacker-controlled +value in the block can leave a constant credential indexed and put the +fields that differ on every request in `no_index`. """. %%%----------------------------------------------------------------------------- @@ -63,6 +97,9 @@ reads a fresh value from the wire. check_name/1, check_value/1, first_invalid/2, + literal_prefix/1, + maybe_insert/3, + name_ref_prefix/2, name_verdict/1, remove_if_seq/3 ]} @@ -75,7 +112,9 @@ reads a fresh value from the wire. is_empty/1, new/0, new/1, + new/2, set_max_table_size/2, + table_entries/1, table_size/1 ]). @@ -99,6 +138,7 @@ reads a fresh value from the wire. encode_opts/0, field_error/0, headers/0, + index_policy/0, state/0 ]). @@ -112,6 +152,21 @@ reads a fresh value from the wire. -type decode_opts() :: #{ max_list_size => pos_integer() | infinity }. + +-doc """ +The field names that the encoder keeps out of the dynamic table. + +`no_index` selects literal without indexing (RFC 7541 §6.2.2) and +`never_index` selects literal never indexed (RFC 7541 §6.2.3). A name in +neither list uses literal with incremental indexing (RFC 7541 §6.2.1). See +the module documentation for the choice between the two lists. +""". +-type index_policy() :: #{ + never_index => [binary()], + no_index => [binary()] +}. + +-type index_rule() :: index | no_index | never_index. -type decode_error() :: dynamic_table_size_exceeded | invalid_table_index @@ -187,7 +242,8 @@ from the wire therefore reads `{invalid_value, _}` as `valid`. oldest_seq = 0 :: non_neg_integer(), entries = #{} :: #{non_neg_integer() => #entry{}}, full_index = #{} :: #{{binary(), binary()} => non_neg_integer()}, - name_index = #{} :: #{binary() => non_neg_integer()} + name_index = #{} :: #{binary() => non_neg_integer()}, + policy = #{} :: #{binary() => no_index | never_index} }). -doc """ @@ -196,6 +252,9 @@ The dynamic table of one HPACK endpoint. `full_index` and `name_index` hold keys for live entries only. An eviction removes the keys that bind to the evicted sequence, so the two indexes stay bounded by the table size. + +`policy` holds the lowercased names of `t:index_policy/0` with the literal +form each one selects. The encoder reads it on the two literal arms only. """. -opaque state() :: #hpack{}. @@ -219,11 +278,28 @@ new() -> new(MaxSize) -> {ok, #hpack{max_size = MaxSize, configured_max_size = MaxSize}}. +-doc """ +Create a new HPACK encoder state with a max size and an indexing policy. + +The policy names the fields that never enter the dynamic table. See the +module documentation for the two literal forms and the sets that fit a +deployment. A name in both lists resolves to `never_index`, and every name +is lowercased. +""". +-spec new(MaxSize :: non_neg_integer(), Policy :: index_policy()) -> {ok, state()}. +new(MaxSize, Policy) -> + {ok, #hpack{max_size = MaxSize, configured_max_size = MaxSize, policy = policy_map(Policy)}}. + -doc "Update the maximum table size (from SETTINGS_HEADER_TABLE_SIZE). Immediately evicts entries if the new size is smaller than current table size.". -spec set_max_table_size(MaxSize :: non_neg_integer(), State :: state()) -> {ok, state()}. set_max_table_size(MaxSize, State) -> {ok, update_table_size(MaxSize, State#hpack{configured_max_size = MaxSize})}. +-doc "Get the number of live entries in the dynamic table.". +-spec table_entries(State :: state()) -> non_neg_integer(). +table_entries(#hpack{entries = Entries}) -> + map_size(Entries). + -doc "Get the current dynamic table size in bytes.". -spec table_size(State :: state()) -> non_neg_integer(). table_size(#hpack{size = Size}) -> @@ -417,20 +493,24 @@ decode_headers(_, _, _, _, _, _) -> {[iodata()], state()}. encode_headers([], State, _, Acc) -> {lists:reverse(Acc), State}; -encode_headers([{Name0, Value} = Header0 | Tail], State, UseHuffman, Acc) -> +encode_headers( + [{Name0, Value} = Header0 | Tail], State = #hpack{policy = Policy}, UseHuffman, Acc +) -> {Name, _} = Header = lower_name(Header0, Name0, Value), case find(Header, State) of {field, Index} -> Encoded = nhttp_int:enc7(Index, 2#1), encode_headers(Tail, State, UseHuffman, [Encoded | Acc]); {name, Index} -> - State2 = insert(Header, State), - Encoded = [nhttp_int:enc6(Index, 2#01) | nhttp_str:encode(Value, UseHuffman)], + Rule = maps:get(Name, Policy, index), + State2 = maybe_insert(Rule, Header, State), + Encoded = [name_ref_prefix(Rule, Index) | nhttp_str:encode(Value, UseHuffman)], encode_headers(Tail, State2, UseHuffman, [Encoded | Acc]); not_found -> - State2 = insert(Header, State), + Rule = maps:get(Name, Policy, index), + State2 = maybe_insert(Rule, Header, State), Encoded = [ - <<2#01:2, 0:6>> + literal_prefix(Rule) | [nhttp_str:encode(Name, UseHuffman) | nhttp_str:encode(Value, UseHuffman)] ], encode_headers(Tail, State2, UseHuffman, [Encoded | Acc]) @@ -644,6 +724,11 @@ insert({Name, Value}, Verdict, State = #hpack{max_size = MaxSize, next_seq = Nex } end. +-spec literal_prefix(index_rule()) -> binary(). +literal_prefix(index) -> <<2#01:2, 0:6>>; +literal_prefix(no_index) -> <<2#0000:4, 0:4>>; +literal_prefix(never_index) -> <<2#0001:4, 0:4>>. + -spec lookup(pos_integer(), state()) -> {ok, {binary(), binary()}, verdict()} | {error, decode_error()}. lookup(1, _) -> @@ -810,10 +895,29 @@ maybe_emit_table_size_update(State0 = #hpack{configured_max_size = MaxSize}) -> State1 = update_table_size(MaxSize, State0#hpack{max_size = MaxSize}), {nhttp_int:enc5(MaxSize, 2#001), State1}. +-spec maybe_insert(index_rule(), {binary(), binary()}, state()) -> state(). +maybe_insert(index, Header, State) -> insert(Header, State); +maybe_insert(no_index, _Header, State) -> State; +maybe_insert(never_index, _Header, State) -> State. + +-spec name_ref_prefix(index_rule(), pos_integer()) -> binary(). +name_ref_prefix(index, Index) -> nhttp_int:enc6(Index, 2#01); +name_ref_prefix(no_index, Index) -> nhttp_int:enc4(Index, 2#0000); +name_ref_prefix(never_index, Index) -> nhttp_int:enc4(Index, 2#0001). + -spec name_verdict(verdict()) -> verdict(). name_verdict({invalid_value, _}) -> valid; name_verdict(Verdict) -> Verdict. +-spec policy_map(index_policy()) -> #{binary() => no_index | never_index}. +policy_map(Policy) -> + NoIndex = maps:get(no_index, Policy, []), + NeverIndex = maps:get(never_index, Policy, []), + maps:from_list( + [{nhttp_headers:lower_field_name(Name), no_index} || Name <- NoIndex] ++ + [{nhttp_headers:lower_field_name(Name), never_index} || Name <- NeverIndex] + ). + -doc """ Remove `Key` from `Index` when the index still binds it to `Seq`. diff --git a/test/nhttp_h2_SUITE.erl b/test/nhttp_h2_SUITE.erl index 280beb4..3f54630 100644 --- a/test/nhttp_h2_SUITE.erl +++ b/test/nhttp_h2_SUITE.erl @@ -46,7 +46,9 @@ Tests RFC 9113 compliance for: settings_ack_sent/1, settings_updates_peer/1, settings_initial_window_size_update/1, - peer_settings_reports_peer_values/1 + peer_settings_reports_peer_values/1, + settings_hpack_index_policy_reaches_encoder/1, + encoder_table_accessors_report_encoder_state/1 ]). -export([ @@ -210,7 +212,9 @@ groups() -> settings_ack_sent, settings_updates_peer, settings_initial_window_size_update, - peer_settings_reports_peer_values + peer_settings_reports_peer_values, + settings_hpack_index_policy_reaches_encoder, + encoder_table_accessors_report_encoder_state ]}, {flow_control, [parallel], [ connection_window_update, @@ -485,6 +489,44 @@ settings_initial_window_size_update(_Config) -> ?assertMatch([{settings, #{initial_window_size := 131072}} | _], Events), ok. +settings_hpack_index_policy_reaches_encoder(_Config) -> + Policy = #{no_index => [<<":path">>, <<"apns-id">>]}, + Conn0 = nhttp_h2:new(client, #{hpack_index_policy => Policy}), + ?assertEqual( + iolist_to_binary(nhttp_h2:preface(nhttp_h2:new(client))), + iolist_to_binary(nhttp_h2:preface(Conn0)) + ), + {ok, Stream1, Conn1} = nhttp_h2:open_stream(Conn0), + {ok, Conn2, _} = nhttp_h2:send_headers(Conn1, Stream1, push_request(<<"/3/device/a">>), fin), + Entries = nhttp_h2:encoder_table_entries(Conn2), + Size = nhttp_h2:encoder_table_size(Conn2), + ?assertEqual(2, Entries), + ?assertEqual(60 + 60, Size), + {ok, Stream2, Conn3} = nhttp_h2:open_stream(Conn2), + {ok, Conn4, _} = nhttp_h2:send_headers(Conn3, Stream2, push_request(<<"/3/device/b">>), fin), + ?assertEqual(Entries, nhttp_h2:encoder_table_entries(Conn4)), + ?assertEqual(Size, nhttp_h2:encoder_table_size(Conn4)). + +push_request(Path) -> + [ + {<<":method">>, <<"POST">>}, + {<<":scheme">>, <<"https">>}, + {<<":authority">>, <<"api.push.apple.com">>}, + {<<":path">>, Path}, + {<<"apns-id">>, Path}, + {<<"content-type">>, <<"application/json">>} + ]. + +encoder_table_accessors_report_encoder_state(_Config) -> + Conn0 = nhttp_h2:new(client), + ?assertEqual(0, nhttp_h2:encoder_table_entries(Conn0)), + ?assertEqual(0, nhttp_h2:encoder_table_size(Conn0)), + {ok, StreamId, Conn1} = nhttp_h2:open_stream(Conn0), + Headers = [{<<":method">>, <<"GET">>}, {<<":path">>, <<"/x">>}, {<<"x-a">>, <<"b">>}], + {ok, Conn2, _} = nhttp_h2:send_headers(Conn1, StreamId, Headers, fin), + ?assertEqual(2, nhttp_h2:encoder_table_entries(Conn2)), + ?assertEqual(39 + 36, nhttp_h2:encoder_table_size(Conn2)). + peer_settings_reports_peer_values(_Config) -> Conn0 = nhttp_h2:new(client), Defaults = nhttp_h2:peer_settings(Conn0), diff --git a/test/nhttp_hpack_SUITE.erl b/test/nhttp_hpack_SUITE.erl index f793d6a..830b709 100644 --- a/test/nhttp_hpack_SUITE.erl +++ b/test/nhttp_hpack_SUITE.erl @@ -24,7 +24,8 @@ all() -> {group, error_handling}, {group, encode_lowercase}, {group, coverage_edge_cases}, - {group, index_maps} + {group, index_maps}, + {group, index_policy} ]. groups() -> @@ -32,8 +33,10 @@ groups() -> {state, [parallel], [ new_default, new_with_size, + new_with_policy, is_empty_initial, - table_size_initial + table_size_initial, + table_entries_initial ]}, {roundtrip, [parallel], [ roundtrip_simple_request, @@ -121,6 +124,18 @@ groups() -> evicted_key_is_absent, rebound_key_survives_eviction, allocation_per_request_is_flat + ]}, + {index_policy, [], [ + no_index_emits_literal_without_indexing, + never_index_emits_literal_never_indexed, + unlisted_name_encodes_as_before, + encode_rfc_c3_sequence_with_policy, + authorization_survives_unique_paths, + path_only_policy_turns_the_table_over, + no_index_keeps_static_name_reference, + table_entries_tracks_insertions_and_evictions, + policy_state_is_flat, + policy_names_are_lowercased_and_never_index_wins ]} ]. @@ -164,6 +179,19 @@ table_size_initial(_Config) -> {ok, State} = nhttp_hpack:new(), ?assertEqual(0, nhttp_hpack:table_size(State)). +new_with_policy(_Config) -> + Policy = #{no_index => [<<":path">>], never_index => [<<"authorization">>]}, + {ok, State} = nhttp_hpack:new(8192, Policy), + ?assertEqual(0, nhttp_hpack:table_size(State)), + ?assertEqual(0, nhttp_hpack:table_entries(State)), + ?assert(nhttp_hpack:is_empty(State)), + {ok, Empty} = nhttp_hpack:new(8192, #{}), + ?assertEqual(0, nhttp_hpack:table_entries(Empty)). + +table_entries_initial(_Config) -> + {ok, State} = nhttp_hpack:new(), + ?assertEqual(0, nhttp_hpack:table_entries(State)). + %%%----------------------------------------------------------------------------- %%% ROUNDTRIP TESTS %%%----------------------------------------------------------------------------- @@ -378,12 +406,14 @@ decode_rfc_c4_3(_Config) -> ?assertEqual(Expected, Decoded). encode_rfc_c3_sequence(_Config) -> - Expected = [ + encode_sequence(#{huffman => false}, rfc_c3_expected()). + +rfc_c3_expected() -> + [ <<16#82, 16#86, 16#84, 16#41, 16#0f, "www.example.com">>, <<16#82, 16#86, 16#84, 16#be, 16#58, 16#08, "no-cache">>, <<16#82, 16#87, 16#85, 16#bf, 16#40, 16#0a, "custom-key", 16#0c, "custom-value">> - ], - encode_sequence(#{huffman => false}, Expected). + ]. encode_rfc_c4_sequence(_Config) -> Expected = [ @@ -399,6 +429,10 @@ encode_rfc_c4_sequence(_Config) -> %% request reuse `:authority' from the dynamic table, so an encoder that only %% consults the static table never reaches the expected octets. encode_sequence(Opts, Expected) -> + {ok, State0} = nhttp_hpack:new(), + encode_sequence(Opts, Expected, State0). + +encode_sequence(Opts, Expected, State0) -> Requests = [ [ {<<":method">>, <<"GET">>}, @@ -421,7 +455,6 @@ encode_sequence(Opts, Expected) -> {<<"custom-key">>, <<"custom-value">>} ] ], - {ok, State0} = nhttp_hpack:new(), lists:foldl( fun({Headers, Want}, State) -> {ok, Encoded, State2} = nhttp_hpack:encode(Headers, State, Opts), @@ -986,6 +1019,155 @@ allocation_per_request_is_flat(_Config) -> ct:log("words allocated: first window ~p, second window ~p", [Words1, Words2]), ?assert(abs(Words2 - Words1) * 100 =< Words1). +%%%----------------------------------------------------------------------------- +%%% INDEX POLICY TESTS +%%%----------------------------------------------------------------------------- + +no_index_emits_literal_without_indexing(_Config) -> + Policy = #{no_index => [<<":path">>, <<"apns-id">>]}, + {ok, State0} = nhttp_hpack:new(4096, Policy), + Headers = [{<<":path">>, <<"/3/device/ab">>}, {<<"apns-id">>, <<"id-1">>}], + {ok, Encoded, State1} = nhttp_hpack:encode(Headers, State0), + ?assertEqual( + <<2#0000:4, 4:4, 12, "/3/device/ab", 2#0000:4, 0:4, 7, "apns-id", 4, "id-1">>, + iolist_to_binary(Encoded) + ), + ?assertEqual(0, nhttp_hpack:table_size(State1)), + ?assertEqual(0, nhttp_hpack:table_entries(State1)), + {ok, Decoder0} = nhttp_hpack:new(), + {ok, Decoded, Decoder1} = nhttp_hpack:decode(iolist_to_binary(Encoded), Decoder0), + ?assertEqual(Headers, Decoded), + ?assertEqual(0, nhttp_hpack:table_entries(Decoder1)). + +never_index_emits_literal_never_indexed(_Config) -> + Policy = #{never_index => [<<"authorization">>, <<"x-secret">>]}, + {ok, State0} = nhttp_hpack:new(4096, Policy), + Headers = [{<<"authorization">>, <<"bearer t">>}, {<<"x-secret">>, <<"s">>}], + {ok, Encoded, State1} = nhttp_hpack:encode(Headers, State0), + ?assertEqual( + <<2#0001:4, 15:4, 8, 8, "bearer t", 2#0001:4, 0:4, 8, "x-secret", 1, "s">>, + iolist_to_binary(Encoded) + ), + ?assertEqual(0, nhttp_hpack:table_size(State1)), + ?assertEqual(0, nhttp_hpack:table_entries(State1)), + {ok, Decoder0} = nhttp_hpack:new(), + {ok, Decoded, Decoder1} = nhttp_hpack:decode(iolist_to_binary(Encoded), Decoder0), + ?assertEqual(Headers, Decoded), + ?assertEqual(0, nhttp_hpack:table_entries(Decoder1)). + +unlisted_name_encodes_as_before(_Config) -> + Policy = #{no_index => [<<":path">>], never_index => [<<"authorization">>]}, + {ok, Plain0} = nhttp_hpack:new(4096), + {ok, Policed0} = nhttp_hpack:new(4096, Policy), + Headers = [ + Header + || {Name, _} = Header <- apns_request(1), + Name =/= <<":path">>, + Name =/= <<"authorization">> + ], + {ok, Plain1, PlainState1} = nhttp_hpack:encode(Headers, Plain0), + {ok, Policed1, PolicedState1} = nhttp_hpack:encode(Headers, Policed0), + ?assertEqual(iolist_to_binary(Plain1), iolist_to_binary(Policed1)), + {ok, Plain2, PlainState2} = nhttp_hpack:encode(Headers, PlainState1), + {ok, Policed2, PolicedState2} = nhttp_hpack:encode(Headers, PolicedState1), + ?assertEqual(iolist_to_binary(Plain2), iolist_to_binary(Policed2)), + ?assertEqual(nhttp_hpack:table_size(PlainState2), nhttp_hpack:table_size(PolicedState2)), + ?assertEqual( + nhttp_hpack:table_entries(PlainState2), nhttp_hpack:table_entries(PolicedState2) + ). + +encode_rfc_c3_sequence_with_policy(_Config) -> + {ok, Empty} = nhttp_hpack:new(4096, #{}), + encode_sequence(#{huffman => false}, rfc_c3_expected(), Empty), + Policy = #{no_index => [<<"apns-id">>], never_index => [<<"authorization">>]}, + {ok, Absent} = nhttp_hpack:new(4096, Policy), + encode_sequence(#{huffman => false}, rfc_c3_expected(), Absent). + +authorization_survives_unique_paths(_Config) -> + {ok, State0} = nhttp_hpack:new(4096, #{no_index => [<<":path">>, <<"apns-id">>]}), + ?assertEqual(none, first_literal_authorization(1000, State0)). + +%% HPACK evicts in insertion order (RFC 7541 Section 4.4), so one unique field +%% that still inserts pushes every constant entry out once the table fills. +path_only_policy_turns_the_table_over(_Config) -> + {ok, State0} = nhttp_hpack:new(4096, #{no_index => [<<":path">>]}), + {literal, N} = first_literal_authorization(1000, State0), + ct:log("authorization re-emitted as a literal at request ~p", [N]), + ?assert(N > 2). + +first_literal_authorization(To, State0) -> + Positions = record_positions(hpack), + {ok, _, State1} = nhttp_hpack:encode(apns_request(1), State0), + {_, Value} = lists:keyfind(<<"authorization">>, 1, apns_request(1)), + Seq = maps:get({<<"authorization">>, Value}, element(maps:get(full_index, Positions), State1)), + first_literal_authorization(2, To, State1, Seq, Positions). + +first_literal_authorization(From, To, _State, _Seq, _Positions) when From > To -> + none; +first_literal_authorization(N, To, State, Seq, Positions) -> + Headers = apns_request(N), + {ok, [_Prefix | PerHeader], State1} = nhttp_hpack:encode(Headers, State), + ?assertEqual(length(Headers), length(PerHeader)), + Names = [Name || {Name, _} <- Headers], + Position = length(lists:takewhile(fun(Name) -> Name =/= <<"authorization">> end, Names)) + 1, + Octets = iolist_to_binary(lists:nth(Position, PerHeader)), + {_, Value} = lists:keyfind(<<"authorization">>, 1, Headers), + FullIndex = element(maps:get(full_index, Positions), State1), + case {is_indexed_field(Octets), maps:get({<<"authorization">>, Value}, FullIndex, none)} of + {true, Seq} -> first_literal_authorization(N + 1, To, State1, Seq, Positions); + _ -> {literal, N} + end. + +is_indexed_field(<<2#1:1, Index:7>>) -> Index >= 62 andalso Index =< 126; +is_indexed_field(<<2#1:1, 127:7, _:8>>) -> true; +is_indexed_field(_) -> false. + +no_index_keeps_static_name_reference(_Config) -> + {ok, State0} = nhttp_hpack:new(4096, #{no_index => [<<":path">>]}), + Path = <<"/3/device/0123456789abcdef">>, + {ok, Encoded, State1} = nhttp_hpack:encode([{<<":path">>, Path}], State0), + ?assertEqual(<<2#0000:4, 4:4, 26, Path/binary>>, iolist_to_binary(Encoded)), + {ok, Root, State2} = nhttp_hpack:encode([{<<":path">>, <<"/">>}], State1), + ?assertEqual(<<2#1:1, 4:7>>, iolist_to_binary(Root)), + ?assertEqual(0, nhttp_hpack:table_entries(State2)). + +table_entries_tracks_insertions_and_evictions(_Config) -> + {ok, State0} = nhttp_hpack:new(72), + {ok, _, State1} = nhttp_hpack:encode([{<<"aa">>, <<"bb">>}], State0), + ?assertEqual({1, 36}, {nhttp_hpack:table_entries(State1), nhttp_hpack:table_size(State1)}), + {ok, _, State2} = nhttp_hpack:encode([{<<"cc">>, <<"dd">>}], State1), + ?assertEqual({2, 72}, {nhttp_hpack:table_entries(State2), nhttp_hpack:table_size(State2)}), + {ok, _, State3} = nhttp_hpack:encode([{<<"ee">>, <<"ff">>}], State2), + ?assertEqual({2, 72}, {nhttp_hpack:table_entries(State3), nhttp_hpack:table_size(State3)}), + {ok, State4} = nhttp_hpack:set_max_table_size(36, State3), + ?assertEqual({1, 36}, {nhttp_hpack:table_entries(State4), nhttp_hpack:table_size(State4)}), + {ok, State5} = nhttp_hpack:set_max_table_size(0, State4), + ?assertEqual({0, 0}, {nhttp_hpack:table_entries(State5), nhttp_hpack:table_size(State5)}). + +policy_state_is_flat(_Config) -> + {ok, Both0} = nhttp_hpack:new(4096, #{no_index => [<<":path">>, <<"apns-id">>]}), + {BothWords1, Both1} = max_state_words(1, 1, Both0), + {BothWords2, Both2} = max_state_words(2, 5000, Both1), + {BothWords3, _Both3} = max_state_words(5001, 10000, Both2), + ct:log("state words with both names unindexed: ~p", [[BothWords1, BothWords2, BothWords3]]), + ?assertEqual(BothWords1, BothWords2), + ?assertEqual(BothWords1, BothWords3), + {ok, Path0} = nhttp_hpack:new(4096, #{no_index => [<<":path">>]}), + {PathWords1, Path1} = max_state_words(1, 1000, Path0), + {PathWords2, _Path2} = max_state_words(1001, 10000, Path1), + ct:log("state words with the path unindexed: ~p", [[PathWords1, PathWords2]]), + ?assert(abs(PathWords2 - PathWords1) =< erts_debug:flat_size(apns_request(1))). + +policy_names_are_lowercased_and_never_index_wins(_Config) -> + Policy = #{no_index => [<<"X-Token">>], never_index => [<<"x-token">>]}, + {ok, State0} = nhttp_hpack:new(4096, Policy), + {ok, Encoded, State1} = nhttp_hpack:encode([{<<"X-Token">>, <<"v">>}], State0), + ?assertEqual(<<2#0001:4, 0:4, 7, "x-token", 1, "v">>, iolist_to_binary(Encoded)), + ?assertEqual(0, nhttp_hpack:table_entries(State1)), + {ok, Lower} = nhttp_hpack:new(4096, #{no_index => [<<"X-Path">>]}), + {ok, Plain, _} = nhttp_hpack:encode([{<<"x-path">>, <<"/">>}], Lower), + ?assertEqual(<<2#0000:4, 0:4, 6, "x-path", 1, "/">>, iolist_to_binary(Plain)). + %%%----------------------------------------------------------------------------- %%% HELPERS %%%----------------------------------------------------------------------------- diff --git a/test/nhttp_props_SUITE.erl b/test/nhttp_props_SUITE.erl index 65de035..04650c3 100644 --- a/test/nhttp_props_SUITE.erl +++ b/test/nhttp_props_SUITE.erl @@ -81,7 +81,8 @@ groups() -> hpack_malformed_index_no_crash, hpack_bounded_after_error, hpack_encode_lowercases_names, - hpack_index_maps_hold_live_entries_only + hpack_index_maps_hold_live_entries_only, + hpack_policy_names_never_enter_the_table ]}, {h1_props, [parallel], [ h1_request_roundtrip, @@ -326,6 +327,9 @@ hpack_encode_lowercases_names(Config) -> hpack_index_maps_hold_live_entries_only(Config) -> run_property(nhttp_hpack_props, prop_index_maps_hold_live_entries_only, Config). +hpack_policy_names_never_enter_the_table(Config) -> + run_property(nhttp_hpack_props, prop_policy_names_never_enter_the_table, Config). + %%%----------------------------------------------------------------------------- %%% HTTP/1.1 PROPERTY TESTS %%%----------------------------------------------------------------------------- diff --git a/test/property_test/nhttp_hpack_props.erl b/test/property_test/nhttp_hpack_props.erl index 9e3fd4d..8492e29 100644 --- a/test/property_test/nhttp_hpack_props.erl +++ b/test/property_test/nhttp_hpack_props.erl @@ -536,6 +536,55 @@ live_field(Seq, Entries, FieldPosition) -> Entry -> element(FieldPosition, Entry) end. +%% No name of the policy is a key of `name_index' or the name of a key of +%% `full_index' after any encode, and the decoder round trips every block. +-spec prop_policy_names_never_enter_the_table() -> triq:property(). +prop_policy_names_never_enter_the_table() -> + Positions = record_positions(hpack), + ?FORALL( + {HeadersList, Rules}, + {non_empty(list(headers_gen())), list(oneof([index, no_index, never_index]))}, + begin + Blocks = lists:sublist(HeadersList, 10), + Names = lists:usort([Name || Headers <- Blocks, {Name, _} <- Headers]), + Policy = policy_from(Names, Rules), + {ok, EncState0} = nhttp_hpack:new(4096, Policy), + {ok, DecState0} = nhttp_hpack:new(), + PolicyNames = maps:get(no_index, Policy) ++ maps:get(never_index, Policy), + policy_names_absent(Blocks, PolicyNames, EncState0, DecState0, Positions) + end + ). + +%% The rule list runs out on a long name list, and the tail then keeps the +%% default rule. +-spec policy_from([binary()], [index | no_index | never_index]) -> nhttp_hpack:index_policy(). +policy_from(Names, Rules) -> + Padded = Rules ++ lists:duplicate(max(0, length(Names) - length(Rules)), index), + Pairs = lists:zip(Names, lists:sublist(Padded, length(Names))), + #{ + no_index => [Name || {Name, no_index} <- Pairs], + never_index => [Name || {Name, never_index} <- Pairs] + }. + +-spec policy_names_absent( + [[{binary(), binary()}]], [binary()], nhttp_hpack:state(), nhttp_hpack:state(), map() +) -> boolean(). +policy_names_absent([], _PolicyNames, _EncState, _DecState, _Positions) -> + true; +policy_names_absent([Headers | Tail], PolicyNames, EncState, DecState, Positions) -> + {ok, Encoded, EncState1} = nhttp_hpack:encode(Headers, EncState), + FullIndex = element(maps:get(full_index, Positions), EncState1), + NameIndex = element(maps:get(name_index, Positions), EncState1), + TableNames = [Name || {Name, _} <- maps:keys(FullIndex)] ++ maps:keys(NameIndex), + Absent = lists:all(fun(Name) -> not lists:member(Name, TableNames) end, PolicyNames), + case nhttp_hpack:decode(iolist_to_binary(Encoded), DecState) of + {ok, Headers, DecState1} -> + Absent andalso + policy_names_absent(Tail, PolicyNames, EncState1, DecState1, Positions); + _ -> + false + end. + -spec record_positions(atom()) -> #{atom() => pos_integer()}. record_positions(Record) -> Beam = filename:join([code:lib_dir(nhttp_lib), "ebin", "nhttp_hpack.beam"]),