feat(hpack): add an indexing policy to the encoder

CHANGELOG.md

@@ -61,6 +61,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `nhttp_h2:stream_send_window/2` reads the send window of one stream
 - `nhttp_h2:peer_settings/1` reads the settings the peer sent, merged over
   the defaults
+- `nhttp_hpack:new/2` takes an indexing policy of type
+  `t:nhttp_hpack:index_policy/0`. A field name in `no_index` encodes as a
+  literal without indexing (RFC 7541 Section 6.2.2) and a name in
+  `never_index` as a literal never indexed (RFC 7541 Section 6.2.3). Neither
+  enters the dynamic table, and both keep a static name reference. The
+  default of `new/0` and `new/1` is the empty policy, which changes no byte
+  of output
+- `hpack_index_policy` on `t:nhttp_h2:settings/0` carries the policy to the
+  HPACK encoder of the connection. It has no wire representation
+- `nhttp_hpack:table_entries/1`, `nhttp_h2:encoder_table_entries/1` and
+  `nhttp_h2:encoder_table_size/1` read the live entry count and the octets
+  of the encoder dynamic table
 
 ### Changed
 

src/nhttp_h2.erl

@@ -101,6 +101,8 @@ Flow-control and END_STREAM ride on the DATA frame.
 %%%-----------------------------------------------------------------------------
 -export([
     connection_send_window/1,
+    encoder_table_entries/1,
+    encoder_table_size/1,
     peer_settings/1,
     stream_send_window/2
 ]).
@@ -139,9 +141,13 @@ Flow-control and END_STREAM ride on the DATA frame.
 Connection settings.
 
 Most keys map to a SETTINGS parameter of RFC 9113 Section 6.5.2 and go on
-the wire. `max_continuation_frames` and `max_reset_streams` are local
-policy. They bound work that a peer can request, they have no wire
-representation, and the encoder drops them.
+the wire. `max_continuation_frames`, `max_reset_streams` and
+`hpack_index_policy` are local policy. They have no wire representation,
+and the encoder drops them. The first two bound work that a peer can
+request. `hpack_index_policy` reaches `nhttp_hpack:new/2` and names the
+fields that the HPACK encoder keeps out of its dynamic table. The
+`nhttp_hpack` module documentation gives the two literal forms and the
+sets that fit a deployment.
 
 `max_continuation_frames` bounds the number of CONTINUATION frames in one
 field section, per RFC 9113 Section 10.5. An empty CONTINUATION frame adds
@@ -161,7 +167,8 @@ CONTINUATION frames that carry the larger field section.
     max_header_list_size => pos_integer() | infinity,
     max_continuation_frames => pos_integer() | infinity,
     enable_connect_protocol => boolean(),
-    max_reset_streams => pos_integer() | infinity
+    max_reset_streams => pos_integer() | infinity,
+    hpack_index_policy => nhttp_hpack:index_policy()
 }.
 
 -type stream_state() ::
@@ -298,7 +305,8 @@ new(Role) ->
 new(Role, LocalSettings) ->
     MergedSettings = maps:merge(default_settings(), LocalSettings),
     HeaderTableSize = maps:get(header_table_size, MergedSettings, ?H2_DEFAULT_HEADER_TABLE_SIZE),
-    {ok, HpackEnc} = nhttp_hpack:new(HeaderTableSize),
+    IndexPolicy = maps:get(hpack_index_policy, MergedSettings, #{}),
+    {ok, HpackEnc} = nhttp_hpack:new(HeaderTableSize, IndexPolicy),
     {ok, HpackDec} = nhttp_hpack:new(HeaderTableSize),
     #h2_conn{
         role = Role,
@@ -577,6 +585,29 @@ takes it positive again.
 connection_send_window(#h2_conn{send_window = Window}) ->
     Window.
 
+-doc """
+Return the number of live entries in the dynamic table of the HPACK encoder.
+
+The count moves with every field that `send_headers/4` inserts and with
+every eviction that the insert forces (RFC 7541 Section 4.4). A field whose
+name is in `hpack_index_policy` never inserts, so a caller that tunes the
+policy reads its effect here.
+""".
+-spec encoder_table_entries(conn()) -> non_neg_integer().
+encoder_table_entries(#h2_conn{hpack_enc = HpackEnc}) ->
+    nhttp_hpack:table_entries(HpackEnc).
+
+-doc """
+Return the size of the dynamic table of the HPACK encoder in octets.
+
+The size counts every live entry as name, value and the 32 octets of
+overhead of RFC 7541 Section 4.1. It never exceeds the
+SETTINGS_HEADER_TABLE_SIZE that the peer sent.
+""".
+-spec encoder_table_size(conn()) -> non_neg_integer().
+encoder_table_size(#h2_conn{hpack_enc = HpackEnc}) ->
+    nhttp_hpack:table_size(HpackEnc).
+
 -doc """
 Return the settings the peer sent, merged over the defaults.
 

src/nhttp_hpack.erl

@@ -52,6 +52,40 @@ field that arrives by index carries that verdict out again and needs no
 second read of the octets. The verdict names the part that failed, so a
 field that reuses the name by index drops a verdict against the value and
 reads a fresh value from the wire.
+
+## Indexing policy
+
+`new/2` takes a policy that names the fields the encoder keeps out of the
+dynamic table. RFC 7541 §6.2 gives three literal representations, and the
+policy selects one for each field name:
+
+- A name in neither list uses literal with incremental indexing (§6.2.1).
+  The field enters the dynamic table of both endpoints.
+- A name in `no_index` uses literal without indexing (§6.2.2). The field
+  enters no table on this hop. Use it for a value that differs on every
+  request. Such a value costs a table entry that no later request reuses.
+- A name in `never_index` uses literal never indexed (§6.2.3). Every
+  intermediary keeps this form on every later hop. Use it for a secret:
+  RFC 7541 §7.1 shows how an attacker who places values in the same block
+  and reads the compressed size recovers an indexed secret, and §7.1.3
+  names this form as the mitigation.
+
+A name in both lists resolves to `never_index`. The policy holds every name
+in lowercase. A name with a static entry keeps its static name reference
+under both literal forms. The default of `new/0` and `new/1` is the empty
+policy, which changes no byte of output.
+
+The dynamic table evicts in insertion order (§4.4). One field that differs
+on every request and still inserts pushes every constant entry out once
+the table fills, and the encoder then re-emits the constant fields as
+literals. A `no_index` list must therefore name every such field.
+
+A client that shares a connection between principals, or that compresses
+a value an attacker controls in the same block as a credential, puts
+`authorization`, `cookie` and `proxy-authorization` in `never_index`. A
+client with one principal on a TLS connection and no attacker-controlled
+value in the block can leave a constant credential indexed and put the
+fields that differ on every request in `no_index`.
 """.
 
 %%%-----------------------------------------------------------------------------
@@ -63,6 +97,9 @@ reads a fresh value from the wire.
         check_name/1,
         check_value/1,
         first_invalid/2,
+        literal_prefix/1,
+        maybe_insert/3,
+        name_ref_prefix/2,
         name_verdict/1,
         remove_if_seq/3
     ]}
@@ -75,7 +112,9 @@ reads a fresh value from the wire.
     is_empty/1,
     new/0,
     new/1,
+    new/2,
     set_max_table_size/2,
+    table_entries/1,
     table_size/1
 ]).
 
@@ -99,6 +138,7 @@ reads a fresh value from the wire.
     encode_opts/0,
     field_error/0,
     headers/0,
+    index_policy/0,
     state/0
 ]).
 
@@ -112,6 +152,21 @@ reads a fresh value from the wire.
 -type decode_opts() :: #{
     max_list_size => pos_integer() | infinity
 }.
+
+-doc """
+The field names that the encoder keeps out of the dynamic table.
+
+`no_index` selects literal without indexing (RFC 7541 §6.2.2) and
+`never_index` selects literal never indexed (RFC 7541 §6.2.3). A name in
+neither list uses literal with incremental indexing (RFC 7541 §6.2.1). See
+the module documentation for the choice between the two lists.
+""".
+-type index_policy() :: #{
+    never_index => [binary()],
+    no_index => [binary()]
+}.
+
+-type index_rule() :: index | no_index | never_index.
 -type decode_error() ::
     dynamic_table_size_exceeded
     | invalid_table_index
@@ -187,7 +242,8 @@ from the wire therefore reads `{invalid_value, _}` as `valid`.
     oldest_seq = 0 :: non_neg_integer(),
     entries = #{} :: #{non_neg_integer() => #entry{}},
     full_index = #{} :: #{{binary(), binary()} => non_neg_integer()},
-    name_index = #{} :: #{binary() => non_neg_integer()}
+    name_index = #{} :: #{binary() => non_neg_integer()},
+    policy = #{} :: #{binary() => no_index | never_index}
 }).
 
 -doc """
@@ -196,6 +252,9 @@ The dynamic table of one HPACK endpoint.
 `full_index` and `name_index` hold keys for live entries only. An eviction
 removes the keys that bind to the evicted sequence, so the two indexes stay
 bounded by the table size.
+
+`policy` holds the lowercased names of `t:index_policy/0` with the literal
+form each one selects. The encoder reads it on the two literal arms only.
 """.
 -opaque state() :: #hpack{}.
 
@@ -219,11 +278,28 @@ new() ->
 new(MaxSize) ->
     {ok, #hpack{max_size = MaxSize, configured_max_size = MaxSize}}.
 
+-doc """
+Create a new HPACK encoder state with a max size and an indexing policy.
+
+The policy names the fields that never enter the dynamic table. See the
+module documentation for the two literal forms and the sets that fit a
+deployment. A name in both lists resolves to `never_index`, and every name
+is lowercased.
+""".
+-spec new(MaxSize :: non_neg_integer(), Policy :: index_policy()) -> {ok, state()}.
+new(MaxSize, Policy) ->
+    {ok, #hpack{max_size = MaxSize, configured_max_size = MaxSize, policy = policy_map(Policy)}}.
+
 -doc "Update the maximum table size (from SETTINGS_HEADER_TABLE_SIZE). Immediately evicts entries if the new size is smaller than current table size.".
 -spec set_max_table_size(MaxSize :: non_neg_integer(), State :: state()) -> {ok, state()}.
 set_max_table_size(MaxSize, State) ->
     {ok, update_table_size(MaxSize, State#hpack{configured_max_size = MaxSize})}.
 
+-doc "Get the number of live entries in the dynamic table.".
+-spec table_entries(State :: state()) -> non_neg_integer().
+table_entries(#hpack{entries = Entries}) ->
+    map_size(Entries).
+
 -doc "Get the current dynamic table size in bytes.".
 -spec table_size(State :: state()) -> non_neg_integer().
 table_size(#hpack{size = Size}) ->
@@ -417,20 +493,24 @@ decode_headers(_, _, _, _, _, _) ->
     {[iodata()], state()}.
 encode_headers([], State, _, Acc) ->
     {lists:reverse(Acc), State};
-encode_headers([{Name0, Value} = Header0 | Tail], State, UseHuffman, Acc) ->
+encode_headers(
+    [{Name0, Value} = Header0 | Tail], State = #hpack{policy = Policy}, UseHuffman, Acc
+) ->
     {Name, _} = Header = lower_name(Header0, Name0, Value),
     case find(Header, State) of
         {field, Index} ->
             Encoded = nhttp_int:enc7(Index, 2#1),
             encode_headers(Tail, State, UseHuffman, [Encoded | Acc]);
         {name, Index} ->
-            State2 = insert(Header, State),
-            Encoded = [nhttp_int:enc6(Index, 2#01) | nhttp_str:encode(Value, UseHuffman)],
+            Rule = maps:get(Name, Policy, index),
+            State2 = maybe_insert(Rule, Header, State),
+            Encoded = [name_ref_prefix(Rule, Index) | nhttp_str:encode(Value, UseHuffman)],
             encode_headers(Tail, State2, UseHuffman, [Encoded | Acc]);
         not_found ->
-            State2 = insert(Header, State),
+            Rule = maps:get(Name, Policy, index),
+            State2 = maybe_insert(Rule, Header, State),
             Encoded = [
-                <<2#01:2, 0:6>>
+                literal_prefix(Rule)
                 | [nhttp_str:encode(Name, UseHuffman) | nhttp_str:encode(Value, UseHuffman)]
             ],
             encode_headers(Tail, State2, UseHuffman, [Encoded | Acc])
@@ -644,6 +724,11 @@ insert({Name, Value}, Verdict, State = #hpack{max_size = MaxSize, next_seq = Nex
             }
     end.
 
+-spec literal_prefix(index_rule()) -> binary().
+literal_prefix(index) -> <<2#01:2, 0:6>>;
+literal_prefix(no_index) -> <<2#0000:4, 0:4>>;
+literal_prefix(never_index) -> <<2#0001:4, 0:4>>.
+
 -spec lookup(pos_integer(), state()) ->
     {ok, {binary(), binary()}, verdict()} | {error, decode_error()}.
 lookup(1, _) ->
@@ -810,10 +895,29 @@ maybe_emit_table_size_update(State0 = #hpack{configured_max_size = MaxSize}) ->
     State1 = update_table_size(MaxSize, State0#hpack{max_size = MaxSize}),
     {nhttp_int:enc5(MaxSize, 2#001), State1}.
 
+-spec maybe_insert(index_rule(), {binary(), binary()}, state()) -> state().
+maybe_insert(index, Header, State) -> insert(Header, State);
+maybe_insert(no_index, _Header, State) -> State;
+maybe_insert(never_index, _Header, State) -> State.
+
+-spec name_ref_prefix(index_rule(), pos_integer()) -> binary().
+name_ref_prefix(index, Index) -> nhttp_int:enc6(Index, 2#01);
+name_ref_prefix(no_index, Index) -> nhttp_int:enc4(Index, 2#0000);
+name_ref_prefix(never_index, Index) -> nhttp_int:enc4(Index, 2#0001).
+
 -spec name_verdict(verdict()) -> verdict().
 name_verdict({invalid_value, _}) -> valid;
 name_verdict(Verdict) -> Verdict.
 
+-spec policy_map(index_policy()) -> #{binary() => no_index | never_index}.
+policy_map(Policy) ->
+    NoIndex = maps:get(no_index, Policy, []),
+    NeverIndex = maps:get(never_index, Policy, []),
+    maps:from_list(
+        [{nhttp_headers:lower_field_name(Name), no_index} || Name <- NoIndex] ++
+            [{nhttp_headers:lower_field_name(Name), never_index} || Name <- NeverIndex]
+    ).
+
 -doc """
 Remove `Key` from `Index` when the index still binds it to `Seq`.
 

test/nhttp_h2_SUITE.erl

@@ -46,7 +46,9 @@ Tests RFC 9113 compliance for:
     settings_ack_sent/1,
     settings_updates_peer/1,
     settings_initial_window_size_update/1,
-    peer_settings_reports_peer_values/1
+    peer_settings_reports_peer_values/1,
+    settings_hpack_index_policy_reaches_encoder/1,
+    encoder_table_accessors_report_encoder_state/1
 ]).
 
 -export([
@@ -210,7 +212,9 @@ groups() ->
             settings_ack_sent,
             settings_updates_peer,
             settings_initial_window_size_update,
-            peer_settings_reports_peer_values
+            peer_settings_reports_peer_values,
+            settings_hpack_index_policy_reaches_encoder,
+            encoder_table_accessors_report_encoder_state
         ]},
         {flow_control, [parallel], [
             connection_window_update,
@@ -485,6 +489,44 @@ settings_initial_window_size_update(_Config) ->
     ?assertMatch([{settings, #{initial_window_size := 131072}} | _], Events),
     ok.
 
+settings_hpack_index_policy_reaches_encoder(_Config) ->
+    Policy = #{no_index => [<<":path">>, <<"apns-id">>]},
+    Conn0 = nhttp_h2:new(client, #{hpack_index_policy => Policy}),
+    ?assertEqual(
+        iolist_to_binary(nhttp_h2:preface(nhttp_h2:new(client))),
+        iolist_to_binary(nhttp_h2:preface(Conn0))
+    ),
+    {ok, Stream1, Conn1} = nhttp_h2:open_stream(Conn0),
+    {ok, Conn2, _} = nhttp_h2:send_headers(Conn1, Stream1, push_request(<<"/3/device/a">>), fin),
+    Entries = nhttp_h2:encoder_table_entries(Conn2),
+    Size = nhttp_h2:encoder_table_size(Conn2),
+    ?assertEqual(2, Entries),
+    ?assertEqual(60 + 60, Size),
+    {ok, Stream2, Conn3} = nhttp_h2:open_stream(Conn2),
+    {ok, Conn4, _} = nhttp_h2:send_headers(Conn3, Stream2, push_request(<<"/3/device/b">>), fin),
+    ?assertEqual(Entries, nhttp_h2:encoder_table_entries(Conn4)),
+    ?assertEqual(Size, nhttp_h2:encoder_table_size(Conn4)).
+
+push_request(Path) ->
+    [
+        {<<":method">>, <<"POST">>},
+        {<<":scheme">>, <<"https">>},
+        {<<":authority">>, <<"api.push.apple.com">>},
+        {<<":path">>, Path},
+        {<<"apns-id">>, Path},
+        {<<"content-type">>, <<"application/json">>}
+    ].
+
+encoder_table_accessors_report_encoder_state(_Config) ->
+    Conn0 = nhttp_h2:new(client),
+    ?assertEqual(0, nhttp_h2:encoder_table_entries(Conn0)),
+    ?assertEqual(0, nhttp_h2:encoder_table_size(Conn0)),
+    {ok, StreamId, Conn1} = nhttp_h2:open_stream(Conn0),
+    Headers = [{<<":method">>, <<"GET">>}, {<<":path">>, <<"/x">>}, {<<"x-a">>, <<"b">>}],
+    {ok, Conn2, _} = nhttp_h2:send_headers(Conn1, StreamId, Headers, fin),
+    ?assertEqual(2, nhttp_h2:encoder_table_entries(Conn2)),
+    ?assertEqual(39 + 36, nhttp_h2:encoder_table_size(Conn2)).
+
 peer_settings_reports_peer_values(_Config) ->
     Conn0 = nhttp_h2:new(client),
     Defaults = nhttp_h2:peer_settings(Conn0),