diff --git a/CHANGELOG.md b/CHANGELOG.md index a64da97e..0646d01e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ CHANGELOG ========= -4.1.1 +4.2.0 ------------------ * Fixed decoding of data pointers with offsets of 2 GiB or greater. The @@ -10,6 +10,43 @@ CHANGELOG with an `IllegalArgumentException`. Every record past the 2 GiB boundary was unreachable in databases larger than 2 GiB, which have been supported since 4.0.0. +* Fixed typed decoding when an unknown field contains a four-byte pointer whose + low three control bits are nonzero. The pointer's control bits were + incorrectly interpreted as a payload size, which could make decoding resume + in the wrong place. The decoder also rejects a skipped value whose complete + header declares a payload that extends past the data section. Databases written + by MaxMind tooling were not affected. +* Fixed UTF-8 decoding for strings whose multibyte characters cross a buffer + chunk boundary. The decoder also rejects an incomplete multibyte character at + the end of a string. +* Bounded the resources that the decoder spends on a single decode operation. A + crafted database could nest data-section pointers to shared targets so that + decoding one record cost exponential time and memory, or point many times at + one large value so that decoding materialized far more data than the file + holds. Each decode is now limited to 65,536 decoded or skipped values under + this reader's work accounting, 128 levels of container nesting, and 2 MiB of + encoded string and bytes payload. The value limit follows the MaxMind DB + specification's resource guidance. The lower depth limit, payload limit, and + exact value accounting are specific to this reader. + * Exceeding a limit throws an `InvalidDatabaseException`. + * The decoder rejects a data-section pointer whose target is another pointer, + as required by the MaxMind DB format. + * Metadata decoding uses all limits. Skipped fields use the value and depth + limits without materializing their payload. + * Cached pointer targets retain their logical value, depth, and payload cost. + Reusing a target charges that recorded cost without decoding or materializing + it again. + * The decoder rejects integer encodings wider than the format permits before + reading their payload. + * The decoder reports truncated string and bytes values, and malformed UTF-8 + strings, as invalid database data. + * The decoder checks declared map and array sizes before decoding their + children. It caps collection preallocation so nested crafted sizes cannot + exhaust the heap before a decoder limit rejects them. +* Improved decoder performance and reduced per-lookup allocation. The decoder + now avoids transient value wrappers, short-circuits common collection targets, + and avoids temporary character buffers for most UTF-8 strings while preserving + strict validation. 4.1.0 (2026-05-12) ------------------ diff --git a/UPGRADING.md b/UPGRADING.md index 3feee79d..a7c51297 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -1,3 +1,32 @@ +# Upgrading to 4.2.0 + +## Decoder Resource Limits + +Version 4.2.0 limits the work and memory used by one record or metadata decode. +The decoder rejects an operation that exceeds any of these limits: + +- 65,536 decoded or skipped values under the Java reader's work accounting +- 128 nested maps or arrays +- 2 MiB of encoded string and bytes payload materialized by the decoder + +Each cached pointer target retains its logical value, depth, and payload cost. +Every pointer occurrence consumes that recorded cost. The cache still avoids +decoding or materializing the target again, but cache state does not determine +whether an operation exceeds a limit. + +These limits leave a wide margin above MaxMind-produced records. A custom +database containing an unusually large record that decoded in an earlier +release may now throw `InvalidDatabaseException`. The limits are not +configurable in this release. + +When the decoder constructs a custom `List` or `Map` type through an `int` +constructor, it passes an initial-capacity hint capped at 128 rather than the +full declared collection size. + +The decoder also rejects a data-section pointer whose target is another pointer, +which the MaxMind DB format does not permit. It rejects integer payloads wider +than their format type permits before reading the payload. + # Upgrading to 4.0.0 This guide covers the breaking changes introduced in version 4.0.0 and how to diff --git a/src/main/java/com/maxmind/db/DecodedValue.java b/src/main/java/com/maxmind/db/DecodedValue.java index 5440a684..a6272d85 100644 --- a/src/main/java/com/maxmind/db/DecodedValue.java +++ b/src/main/java/com/maxmind/db/DecodedValue.java @@ -4,7 +4,12 @@ * {@code DecodedValue} is a wrapper for the decoded value. */ public final class DecodedValue { + private static final int PAYLOAD_SHIFT = 8; + private static final int VALUES_SHIFT = 30; + private static final long PAYLOAD_MASK = (1L << 22) - 1; + final Object value; + private long costs; DecodedValue(Object value) { this.value = value; @@ -13,4 +18,39 @@ public final class DecodedValue { Object value() { return value; } + + int values() { + return values(costs()); + } + + static int values(long costs) { + return (int) (costs >>> VALUES_SHIFT); + } + + long payloadBytes() { + return payloadBytes(costs()); + } + + static long payloadBytes(long costs) { + return (costs >>> PAYLOAD_SHIFT) & PAYLOAD_MASK; + } + + int depth() { + return depth(costs()); + } + + static int depth(long costs) { + return (int) (costs & 0xFF); + } + + DecodedValue costs(int values, long payloadBytes, int depth) { + this.costs = ((long) values << VALUES_SHIFT) + | (payloadBytes << PAYLOAD_SHIFT) + | depth; + return this; + } + + long costs() { + return this.costs; + } } diff --git a/src/main/java/com/maxmind/db/Decoder.java b/src/main/java/com/maxmind/db/Decoder.java index 73a337e5..266adba1 100644 --- a/src/main/java/com/maxmind/db/Decoder.java +++ b/src/main/java/com/maxmind/db/Decoder.java @@ -9,6 +9,7 @@ import java.lang.reflect.ParameterizedType; import java.math.BigInteger; import java.net.InetAddress; +import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; import java.nio.charset.Charset; import java.nio.charset.CharsetDecoder; @@ -24,9 +25,11 @@ * * This class CANNOT be shared between threads */ -class Decoder { +class Decoder implements NodeCache.Loader { private static final Charset UTF_8 = StandardCharsets.UTF_8; + private static final ThreadLocal UTF_8_DECODER = + ThreadLocal.withInitial(UTF_8::newDecoder); private static final int[] POINTER_VALUE_OFFSETS = {0, 0, 1 << 11, (1 << 19) + (1 << 11), 0}; @@ -35,9 +38,35 @@ class Decoder { private final NodeCache cache; - private final long pointerBase; + // Per-operation resource limits. The MaxMind DB specification recommends + // depth and value limits, but permits equivalent reader-specific accounting. + // This decoder charges each decoded or skipped value. Each pointer occurrence + // also consumes the logical cost of its target. Cache misses measure that cost, + // and cache hits replay it. This keeps accounting independent of cache state + // rather than following the specification's example flat value count. + // Container depth, together with rejecting illegal pointer-to-pointer values, + // bounds recursive calls. + // The payload limit bounds encoded string and bytes data materialized by + // this Java decoder. + // The lower depth limit leaves room on a 512 KiB thread stack even for + // pointer-backed maps, which use more Java frames per logical container + // than inline values. A Decoder serves one decode operation on one thread, + // so these fields need no synchronization. + private static final int MAX_DEPTH = 128; + private static final int MAX_VALUES = 1 << 16; + private static final long MAX_PAYLOAD_BYTES = 1 << 21; + + // A collection's declared size is its logical child count, but it is not + // proof that the input contains that many decodable children. When deriving + // an initial capacity from it, limit unused capacity on the active recursion + // path. Completed children remain bounded by MAX_VALUES. + private static final int MAX_INITIAL_COLLECTION_CAPACITY = 128; + private int depth; + private int maxDepth = -1; + private int valuesRemaining = MAX_VALUES; + private long payloadRemaining = MAX_PAYLOAD_BYTES; - private final CharsetDecoder utfDecoder = UTF_8.newDecoder(); + private final long pointerBase; private final Buffer buffer; @@ -95,8 +124,6 @@ class Decoder { this.lookupNetwork = lookupNetwork; } - private final NodeCache.Loader cacheLoader = this::decode; - T decode(long offset, Class cls) throws IOException { if (offset >= this.buffer.capacity()) { throw new InvalidDatabaseException( @@ -104,25 +131,20 @@ T decode(long offset, Class cls) throws IOException { + "pointer larger than the database."); } + this.valuesRemaining = MAX_VALUES; + this.payloadRemaining = MAX_PAYLOAD_BYTES; + this.depth = 0; + this.maxDepth = -1; this.buffer.position(offset); - return cls.cast(decode(cls, null).value()); + return cls.cast(decode(cls, null)); } - private DecodedValue decode(CacheKey key) throws IOException { - long offset = key.offset(); - if (offset >= this.buffer.capacity()) { + private Object decode(Class cls, java.lang.reflect.Type genericType) + throws IOException { + if (--this.valuesRemaining < 0) { throw new InvalidDatabaseException( - "The MaxMind DB file's data section contains bad data: " - + "pointer larger than the database."); + "The MaxMind DB file's data section exceeds the maximum number of values"); } - - this.buffer.position(offset); - Class cls = key.cls(); - return decode(cls, key.type()); - } - - private DecodedValue decode(Class cls, java.lang.reflect.Type genericType) - throws IOException { var ctrlByte = 0xFF & this.buffer.get(); var type = Type.fromControlByte(ctrlByte); @@ -163,28 +185,100 @@ private DecodedValue decode(Class cls, java.lang.reflect.Type genericType }; } - return new DecodedValue(this.decodeByType(type, size, cls, genericType)); + return this.decodeByType(type, size, cls, genericType); + } + + private Object decodeTarget(CacheKey key) throws IOException { + long offset = key.offset(); + if (offset >= this.buffer.capacity()) { + throw new InvalidDatabaseException( + "The MaxMind DB file's data section contains bad data: " + + "pointer larger than the database."); + } + // Validate a target when the cache loader decodes it. A target that was + // loaded successfully has already passed this check, so cache hits do + // not need to reread its control byte. + if (Type.fromControlByte(0xFF & this.buffer.get(offset)) == Type.POINTER) { + throw new InvalidDatabaseException( + "The MaxMind DB file's data section contains a pointer to a pointer"); + } + + this.buffer.position(offset); + Class cls = key.cls(); + return decode(cls, key.type()); + } + + @Override + public DecodedValue load(CacheKey key) throws IOException { + var valuesRemaining = this.valuesRemaining; + var payloadRemaining = this.payloadRemaining; + var depth = this.depth; + var maxDepth = this.maxDepth; + this.maxDepth = depth; + try { + var value = this.decodeTarget(key); + return new DecodedValue(value).costs( + valuesRemaining - this.valuesRemaining, + payloadRemaining - this.payloadRemaining, + this.maxDepth - depth + ); + } finally { + this.valuesRemaining = valuesRemaining; + this.payloadRemaining = payloadRemaining; + this.depth = depth; + this.maxDepth = maxDepth; + } } - DecodedValue decodePointer(long pointer, Class cls, java.lang.reflect.Type genericType) + Object decodePointer(long pointer, Class cls, java.lang.reflect.Type genericType) throws IOException { var position = buffer.position(); var key = new CacheKey<>(pointer, cls, genericType); - DecodedValue value; - if (requiresLookupContext(cls)) { - value = this.decode(key); + Object value; + if (this.cache == NoCache.getInstance() || requiresLookupContext(cls)) { + value = this.decodeTarget(key); } else { - value = cache.get(key, cacheLoader); + var decodedValue = cache.get(key, this); + this.charge(decodedValue); + value = decodedValue.value(); } buffer.position(position); return value; } + private void charge(DecodedValue value) throws InvalidDatabaseException { + var costs = value.costs(); + var values = DecodedValue.values(costs); + var payloadBytes = DecodedValue.payloadBytes(costs); + var depth = DecodedValue.depth(costs); + var valuesRemaining = this.valuesRemaining - values; + var payloadRemaining = this.payloadRemaining - payloadBytes; + + if (valuesRemaining < 0) { + throw new InvalidDatabaseException( + "The MaxMind DB file's data section exceeds the maximum number of values"); + } + if (payloadRemaining < 0) { + throw new InvalidDatabaseException( + "The MaxMind DB file's data section exceeds the maximum payload size"); + } + if (depth > MAX_DEPTH - this.depth) { + throw new InvalidDatabaseException( + "The MaxMind DB file's data section exceeds the maximum depth"); + } + + this.valuesRemaining = valuesRemaining; + this.payloadRemaining = payloadRemaining; + if (this.maxDepth >= 0) { + this.maxDepth = Math.max(this.maxDepth, this.depth + depth); + } + } + private boolean requiresLookupContext(Class cls) { if (cls == null - || cls.equals(Object.class) + || cls == Object.class || Map.class.isAssignableFrom(cls) || List.class.isAssignableFrom(cls) || cls.isEnum() @@ -209,11 +303,77 @@ private static boolean isSimpleType(Class cls) { if (cls.isPrimitive() || cls.isArray()) { return true; } - return cls.equals(String.class) + return cls == String.class || Number.class.isAssignableFrom(cls) - || cls.equals(Boolean.class) - || cls.equals(Character.class) - || cls.equals(BigInteger.class); + || cls == Boolean.class + || cls == Character.class + || cls == BigInteger.class; + } + + // A container cannot hold more entries than there are bytes left to encode + // them: every key, value, and element occupies at least one byte. Reject an + // impossible declared size before it is used as an allocation hint, so a + // tiny crafted database cannot force a huge list or map preallocation and + // exhaust memory. valueCount is the number of encoded values the container + // declares (an array of N declares N, a map of N declares 2N). + private void checkContainerSize(long valueCount) throws InvalidDatabaseException { + // A container cannot decode more values than the per-operation budget + // allows, so reject an oversized declaration before allocating for it + // rather than after the per-value limit stops the decode. + if (valueCount > this.valuesRemaining) { + throw new InvalidDatabaseException( + "The MaxMind DB file's data section exceeds the maximum number of values"); + } + if (valueCount > this.buffer.capacity() - this.buffer.position()) { + throw new InvalidDatabaseException( + "The MaxMind DB file's data section contains bad data: " + + "a container declares more entries than the data section can hold"); + } + } + + private void enterContainer(long valueCount) throws InvalidDatabaseException { + this.checkContainerSize(valueCount); + if (this.depth >= MAX_DEPTH) { + throw new InvalidDatabaseException( + "The MaxMind DB file's data section exceeds the maximum depth"); + } + this.depth++; + if (this.maxDepth >= 0) { + this.maxDepth = Math.max(this.maxDepth, this.depth); + } + } + + // Charge a string or bytes payload against the per-operation budget before + // materializing it. A payload amplification points many pointers at one large + // value. Cached targets retain their logical payload cost, so each pointer + // occurrence consumes the cost even when the decoder reuses the value. The + // comparison is against the remaining budget so it cannot overflow. The + // limit is inclusive: a total exactly at the limit is allowed. + private void chargePayload(long length) throws InvalidDatabaseException { + if (length > this.payloadRemaining) { + throw new InvalidDatabaseException( + "The MaxMind DB file's data section exceeds the maximum payload size"); + } + this.checkDataSize(length); + this.payloadRemaining -= length; + } + + private void checkDataSize(long length) throws InvalidDatabaseException { + if (length > this.buffer.capacity() - this.buffer.position()) { + throw new InvalidDatabaseException( + "The MaxMind DB file's data section contains bad data: " + + "a value extends beyond the end of the data section."); + } + } + + private static int initialMapCapacity(int size) { + // HashMap's constructor argument is a table capacity rather than an + // expected entry count. Account for its default 0.75 load factor when + // that can be done without exceeding the allocation-hint limit. + return Math.min( + size + (size + 2) / 3, + MAX_INITIAL_COLLECTION_CAPACITY + ); } private Object decodeByType( @@ -223,8 +383,14 @@ private Object decodeByType( java.lang.reflect.Type genericType ) throws IOException { switch (type) { - case MAP: - return this.decodeMap(size, cls, genericType); + case MAP: { + this.enterContainer((long) size * 2); + try { + return this.decodeMap(size, cls, genericType); + } finally { + this.depth--; + } + } case ARRAY: Class elementClass = Object.class; if (genericType instanceof ParameterizedType ptype) { @@ -233,7 +399,12 @@ private Object decodeByType( elementClass = (Class) actualTypes[0]; } } - return this.decodeArray(size, cls, elementClass); + this.enterContainer(size); + try { + return this.decodeArray(size, cls, elementClass); + } finally { + this.depth--; + } case BOOLEAN: Boolean bool = Decoder.decodeBoolean(size); return convertValue(bool, cls); @@ -247,27 +418,46 @@ private Object decodeByType( case BYTES: return this.getByteArray(size); case UINT16: + this.checkIntegerSize("uint16", size, 2); return coerceFromInt(this.decodeUint16(size), cls); case UINT32: + this.checkIntegerSize("uint32", size, 4); return coerceFromLong(this.decodeUint32(size), cls); case INT32: + this.checkIntegerSize("int32", size, 4); return coerceFromInt(this.decodeInt32(size), cls); case UINT64: + this.checkIntegerSize("uint64", size, 8); + return this.decodeLargeUint(size, cls); case UINT128: - // Optimization: for typed fields, avoid BigInteger allocation when - // value fits in long. Keep Object.class behavior unchanged for - // backward compatibility. - if (size < 8 && !cls.equals(Object.class)) { - return coerceFromLong(this.decodeLong(size), cls); - } - // Size >= 8 bytes or Object.class target: use BigInteger - return coerceFromBigInteger(this.decodeBigInteger(size), cls); + this.checkIntegerSize("uint128", size, 16); + return this.decodeLargeUint(size, cls); default: throw new InvalidDatabaseException( "Unknown or unexpected type: " + type.name()); } } + private Object decodeLargeUint(int size, Class cls) + throws InvalidDatabaseException { + // For typed fields, avoid BigInteger allocation when the value fits in + // long. Keep Object.class behavior unchanged for backward compatibility. + if (size < 8 && !cls.equals(Object.class)) { + return coerceFromLong(this.decodeLong(size), cls); + } + return coerceFromBigInteger(this.decodeBigInteger(size), cls); + } + + private void checkIntegerSize(String type, int size, int maximum) + throws InvalidDatabaseException { + if (size > maximum) { + throw new InvalidDatabaseException( + "The MaxMind DB file's data section contains bad data: " + + "invalid size of " + type + "."); + } + this.checkDataSize(size); + } + private static Object coerceFromInt(int value, Class target) { if (target.equals(Object.class) || target.equals(Integer.TYPE) @@ -377,12 +567,26 @@ private static Object coerceFromBigInteger(BigInteger value, Class target) { return value; } - private String decodeString(long size) throws CharacterCodingException { - var oldLimit = buffer.limit(); - buffer.limit(buffer.position() + size); - var s = buffer.decode(utfDecoder); - buffer.limit(oldLimit); - return s; + private String decodeString(long size) throws IOException { + this.chargePayload(size); + // Performance optimization: String's UTF-8 path avoids the temporary + // CharBuffer and char[] used by CharsetDecoder, despite this byte[] copy. + // On OpenJDK 26, random GeoLite2-City lookup throughput improved by about + // 6% with CHMCache and 22% without caching over the previous decoder. + var bytes = new byte[(int) size]; + this.buffer.get(bytes); + var value = new String(bytes, UTF_8); + // String replaces malformed UTF-8 with U+FFFD. Validate strings containing + // that character to distinguish malformed input from a literal U+FFFD. + if (value.indexOf(0xFFFD) >= 0) { + try { + UTF_8_DECODER.get().decode(ByteBuffer.wrap(bytes)); + } catch (CharacterCodingException e) { + throw new InvalidDatabaseException( + "The MaxMind DB file's data section contains an invalid UTF-8 string", e); + } + } + return value; } private int decodeUint16(int size) { @@ -425,8 +629,8 @@ static int decodeInteger(Buffer buffer, int base, int size) { return integer; } - private BigInteger decodeBigInteger(int size) { - var bytes = this.getByteArray(size); + private BigInteger decodeBigInteger(int size) throws InvalidDatabaseException { + var bytes = Decoder.getByteArray(this.buffer, size); return new BigInteger(1, bytes); } @@ -464,13 +668,16 @@ private List decodeArray( Class cls, Class elementClass ) throws IOException { - if (!List.class.isAssignableFrom(cls) && !cls.equals(Object.class)) { + if (cls != Object.class + && cls != List.class + && !List.class.isAssignableFrom(cls)) { throw new DeserializationException("Unable to deserialize an array into an " + cls); } List array; - if (cls.equals(List.class) || cls.equals(Object.class)) { - array = new ArrayList<>(size); + var initialCapacity = Math.min(size, MAX_INITIAL_COLLECTION_CAPACITY); + if (cls == List.class || cls == Object.class) { + array = new ArrayList<>(initialCapacity); } else { Constructor constructor; try { @@ -479,7 +686,7 @@ private List decodeArray( throw new DeserializationException( "No constructor found for the List: " + e.getMessage(), e); } - var parameters = new Object[]{size}; + var parameters = new Object[]{initialCapacity}; try { @SuppressWarnings("unchecked") var array2 = (List) constructor.newInstance(parameters); @@ -492,7 +699,7 @@ private List decodeArray( } for (int i = 0; i < size; i++) { - var e = this.decode(elementClass, null).value(); + var e = this.decode(elementClass, null); array.add(elementClass.cast(e)); } @@ -504,13 +711,13 @@ private Object decodeMap( Class cls, java.lang.reflect.Type genericType ) throws IOException { - if (Map.class.isAssignableFrom(cls) || cls.equals(Object.class)) { + if (cls == Object.class || cls == Map.class || Map.class.isAssignableFrom(cls)) { Class valueClass = Object.class; if (genericType instanceof ParameterizedType ptype) { var actualTypes = ptype.getActualTypeArguments(); if (actualTypes.length == 2) { var keyClass = (Class) actualTypes[0]; - if (!keyClass.equals(String.class)) { + if (keyClass != String.class) { throw new DeserializationException("Map keys must be strings."); } @@ -529,9 +736,10 @@ private Map decodeMapIntoMap( Class valueClass ) throws IOException { Map map; - if (cls.equals(Map.class) || cls.equals(Object.class)) { - map = new HashMap<>(size); + if (cls == Map.class || cls == Object.class) { + map = new HashMap<>(initialMapCapacity(size)); } else { + var initialCapacity = Math.min(size, MAX_INITIAL_COLLECTION_CAPACITY); Constructor constructor; try { constructor = cls.getConstructor(Integer.TYPE); @@ -539,7 +747,7 @@ private Map decodeMapIntoMap( throw new DeserializationException( "No constructor found for the Map: " + e.getMessage(), e); } - var parameters = new Object[]{size}; + var parameters = new Object[]{initialCapacity}; try { @SuppressWarnings("unchecked") var map2 = (Map) constructor.newInstance(parameters); @@ -552,8 +760,8 @@ private Map decodeMapIntoMap( } for (int i = 0; i < size; i++) { - var key = (String) this.decode(String.class, null).value(); - var value = this.decode(valueClass, null).value(); + var key = (String) this.decode(String.class, null); + var value = this.decode(valueClass, null); try { map.put(key, valueClass.cast(value)); } catch (ClassCastException e) { @@ -664,7 +872,7 @@ private Object decodeMapIntoObject(int size, Class cls) var parameters = new Object[parameterTypes.length]; for (int i = 0; i < size; i++) { - var key = (String) this.decode(String.class, null).value(); + var key = (String) this.decode(String.class, null); var parameterIndex = parameterIndexes.get(key); if (parameterIndex == null) { @@ -676,7 +884,7 @@ private Object decodeMapIntoObject(int size, Class cls) parameters[parameterIndex] = this.decode( parameterTypes[parameterIndex], parameterGenericTypes[parameterIndex] - ).value(); + ); } for (int i = 0; i < parameters.length; i++) { @@ -1104,35 +1312,74 @@ private static Object parseDefault(String value, Class target) { private long nextValueOffset(long offset, int numberToSkip) throws InvalidDatabaseException { - if (numberToSkip == 0) { - return offset; - } - - var ctrlData = this.getCtrlData(offset); - var ctrlByte = ctrlData.ctrlByte(); - var size = ctrlData.size(); - offset = ctrlData.offset(); + // Iterate over siblings so a large flat unknown value cannot exhaust + // the Java stack. Recursion is only used to track structural nesting, + // which is bounded by the same limit as normal decoding. + for (var i = 0; i < numberToSkip; i++) { + if (--this.valuesRemaining < 0) { + throw new InvalidDatabaseException( + "The MaxMind DB file's data section exceeds the maximum number of values"); + } - var type = ctrlData.type(); - switch (type) { - case POINTER: - var pointerSize = ((ctrlByte >>> 3) & 0x3) + 1; - offset += pointerSize; - break; - case MAP: - numberToSkip += 2 * size; - break; - case ARRAY: - numberToSkip += size; - break; - case BOOLEAN: - break; - default: - offset += size; - break; + var ctrlData = this.getCtrlData(offset); + var ctrlByte = ctrlData.ctrlByte(); + var size = ctrlData.size(); + offset = ctrlData.offset(); + + switch (ctrlData.type()) { + case POINTER: + var pointerSize = ((ctrlByte >>> 3) & 0x3) + 1; + offset += pointerSize; + break; + case MAP: + this.enterContainer((long) size * 2); + try { + offset = this.nextValueOffset(offset, 2 * size); + } finally { + this.depth--; + } + break; + case ARRAY: + this.enterContainer(size); + try { + offset = this.nextValueOffset(offset, size); + } finally { + this.depth--; + } + break; + case BOOLEAN: + break; + case UINT16: + this.checkIntegerSize("uint16", size, 2); + offset += size; + break; + case UINT32: + this.checkIntegerSize("uint32", size, 4); + offset += size; + break; + case INT32: + this.checkIntegerSize("int32", size, 4); + offset += size; + break; + case UINT64: + this.checkIntegerSize("uint64", size, 8); + offset += size; + break; + case UINT128: + this.checkIntegerSize("uint128", size, 16); + offset += size; + break; + default: + offset += size; + break; + } + if (offset > this.buffer.capacity()) { + throw new InvalidDatabaseException( + "The MaxMind DB file's data section contains bad data: " + + "a value extends beyond the end of the data section."); + } } - - return nextValueOffset(offset, numberToSkip - 1); + return offset; } private CtrlData getCtrlData(long offset) @@ -1165,6 +1412,12 @@ private CtrlData getCtrlData(long offset) offset++; } + // Pointer control bits encode pointer width and value bits, not a + // generic payload size. The caller advances by the pointer width. + if (type.equals(Type.POINTER)) { + return new CtrlData(type, ctrlByte, offset, 0); + } + var size = ctrlByte & 0x1f; if (size >= 29) { var bytesToRead = size - 28; @@ -1179,7 +1432,8 @@ private CtrlData getCtrlData(long offset) return new CtrlData(type, ctrlByte, offset, size); } - private byte[] getByteArray(int length) { + private byte[] getByteArray(int length) throws InvalidDatabaseException { + this.chargePayload(length); return Decoder.getByteArray(this.buffer, length); } diff --git a/src/main/java/com/maxmind/db/MultiBuffer.java b/src/main/java/com/maxmind/db/MultiBuffer.java index 16a02014..39f1136e 100644 --- a/src/main/java/com/maxmind/db/MultiBuffer.java +++ b/src/main/java/com/maxmind/db/MultiBuffer.java @@ -2,7 +2,6 @@ import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.CharBuffer; import java.nio.channels.FileChannel; import java.nio.charset.CharacterCodingException; import java.nio.charset.CharsetDecoder; @@ -215,48 +214,43 @@ public String decode(CharsetDecoder decoder) return this.decode(decoder, Integer.MAX_VALUE); } - String decode(CharsetDecoder decoder, int maxCharBufferSize) + String decode(CharsetDecoder decoder, int maximumSize) throws CharacterCodingException { var remainingBytes = limit - position; - // Cannot allocate more than maxCharBufferSize for CharBuffer - if (remainingBytes > maxCharBufferSize) { + if (remainingBytes > maximumSize) { throw new IllegalStateException( - "Decoding region too large to fit in a CharBuffer: " + remainingBytes + "Decoding region exceeds the maximum size: " + remainingBytes ); } - var out = CharBuffer.allocate((int) remainingBytes); - var pos = position; - - while (remainingBytes > 0) { - // Locate which underlying buffer we are in - var bufIndex = (int) (pos / this.chunkSize); - var bufOffset = (int) (pos % this.chunkSize); - - var srcView = buffers[bufIndex]; - var savedLimit = srcView.limit(); - srcView.position(bufOffset); - - var toRead = (int) Math.min(srcView.remaining(), remainingBytes); - srcView.limit(bufOffset + toRead); - - var result = decoder.decode(srcView, out, false); - srcView.limit(savedLimit); + if (remainingBytes == 0) { + return ""; + } - if (result.isError()) { - result.throwException(); + var bufIndex = (int) (position / this.chunkSize); + var bufOffset = (int) (position % this.chunkSize); + var source = buffers[bufIndex]; + if (remainingBytes <= source.limit() - bufOffset) { + var savedLimit = source.limit(); + source.position(bufOffset); + source.limit(bufOffset + (int) remainingBytes); + try { + var value = decoder.decode(source).toString(); + this.position += remainingBytes; + return value; + } finally { + source.limit(savedLimit); } - - pos += toRead; - remainingBytes -= toRead; } - // Update this MultiBuffer’s logical position - this.position = pos; - - out.flip(); - return out.toString(); + var bytes = new byte[(int) remainingBytes]; + var savedPosition = this.position; + this.get(bytes); + this.position = savedPosition; + var value = decoder.decode(ByteBuffer.wrap(bytes)).toString(); + this.position = this.limit; + return value; } /** diff --git a/src/test/java/com/maxmind/db/DecoderTest.java b/src/test/java/com/maxmind/db/DecoderTest.java index c68b1131..a01367a9 100644 --- a/src/test/java/com/maxmind/db/DecoderTest.java +++ b/src/test/java/com/maxmind/db/DecoderTest.java @@ -4,20 +4,36 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertThrows; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Test; @SuppressWarnings({"boxing", "static-method"}) public class DecoderTest { + private static final int TEST_MAX_DEPTH = 128; + + @Test + public void testDecodedValueStoresMaximumCosts() { + var value = new DecodedValue(null).costs(1 << 16, 1L << 21, TEST_MAX_DEPTH); + assertEquals(1 << 16, value.values()); + assertEquals(1L << 21, value.payloadBytes()); + assertEquals(TEST_MAX_DEPTH, value.depth()); + } + private static Map int32() { int max = (2 << 30) - 1; var int32 = new HashMap(); @@ -136,6 +152,8 @@ private static Map strings() { DecoderTest.addTestString(strings, (byte) 0x40, ""); DecoderTest.addTestString(strings, (byte) 0x41, "1"); DecoderTest.addTestString(strings, (byte) 0x43, "人"); + DecoderTest.addTestString(strings, (byte) 0x43, "\uFFFD"); + DecoderTest.addTestString(strings, (byte) 0x45, "a\uFFFDz"); DecoderTest.addTestString(strings, (byte) 0x43, "123"); DecoderTest.addTestString(strings, (byte) 0x5b, "123456789012345678901234567"); @@ -353,6 +371,106 @@ public void testUint128() throws IOException { DecoderTest.testTypeDecoding(Type.UINT128, largeUint(128)); } + @Test + public void testOversizedIntegersAreRejectedBeforePayloadRead() { + var invalidIntegers = Map.of( + "uint16", new byte[] {(byte) 0xA3}, + "uint32", new byte[] {(byte) 0xC5}, + "int32", new byte[] {0x05, 0x01}, + "uint64", new byte[] {0x09, 0x02}, + "uint128", new byte[] {0x11, 0x03} + ); + + for (var invalidInteger : invalidIntegers.entrySet()) { + var decoder = new Decoder( + NoCache.getInstance(), + SingleBuffer.wrap(invalidInteger.getValue()), + 0 + ); + var ex = assertThrows( + InvalidDatabaseException.class, + () -> decoder.decode(0, Object.class) + ); + assertThat(ex.getMessage(), containsString( + "invalid size of " + invalidInteger.getKey())); + } + } + + @Test + public void testTruncatedIntegersAreRejectedAsInvalidDatabase() { + var headers = List.of( + new byte[] {(byte) 0xA1}, + new byte[] {(byte) 0xC1}, + new byte[] {0x01, 0x01}, + new byte[] {0x01, 0x02}, + new byte[] {0x01, 0x03} + ); + + for (var header : headers) { + var decoder = new Decoder(NoCache.getInstance(), SingleBuffer.wrap(header), 0); + var ex = assertThrows( + InvalidDatabaseException.class, + () -> decoder.decode(0, Object.class) + ); + assertThat(ex.getMessage(), containsString("extends beyond the end")); + } + } + + @Test + public void testPointerBackedOversizedIntegerIsRejectedBeforePayloadRead() { + // A uint32 control byte can declare a 16,843,036-byte payload. The + // pointer target must be rejected before the decoder enters that loop. + var data = new byte[] { + (byte) 0xDF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, + 0x20, 0x00 + }; + for (var cache : List.of(NoCache.getInstance(), new CHMCache())) { + var decoder = new Decoder(cache, SingleBuffer.wrap(data), 0); + var ex = assertThrows( + InvalidDatabaseException.class, + () -> decoder.decode(4, Object.class) + ); + assertThat(ex.getMessage(), containsString("invalid size of uint32")); + } + } + + @Test + public void testSkippedOversizedIntegersAreRejected() { + var invalidIntegers = Map.of( + "uint16", new byte[] {(byte) 0xA3, 0, 0, 0}, + "uint32", new byte[] {(byte) 0xC5, 0, 0, 0, 0, 0}, + "int32", new byte[] {0x05, 0x01, 0, 0, 0, 0, 0}, + "uint64", new byte[] {0x09, 0x02, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + "uint128", new byte[] { + 0x11, 0x03, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + } + ); + + for (var invalidInteger : invalidIntegers.entrySet()) { + var out = new ByteArrayOutputStream(); + out.write(0xE2); // map with two key/value pairs + out.write(0x47); // seven-byte UTF-8 string + out.writeBytes("unknown".getBytes(StandardCharsets.UTF_8)); + out.writeBytes(invalidInteger.getValue()); + out.write(0x45); // five-byte UTF-8 string + out.writeBytes("known".getBytes(StandardCharsets.UTF_8)); + out.write(0x42); // two-byte UTF-8 string + out.writeBytes("ok".getBytes(StandardCharsets.UTF_8)); + + var decoder = new Decoder( + NoCache.getInstance(), + SingleBuffer.wrap(out.toByteArray()), + 0 + ); + var ex = assertThrows( + InvalidDatabaseException.class, + () -> decoder.decode(0, KnownFieldModel.class) + ); + assertThat(ex.getMessage(), containsString( + "invalid size of " + invalidInteger.getKey())); + } + } + @Test public void testDoubles() throws IOException { DecoderTest @@ -375,6 +493,69 @@ public void testStrings() throws IOException { DecoderTest.strings()); } + @Test + public void testUtf8PointerAcrossChunks() throws IOException { + var expected = "a€𐍈\uFFFDz"; + var payload = expected.getBytes(StandardCharsets.UTF_8); + for (int chunkSize : new int[] {1, 2, 3, 4, 5, 64}) { + for (var cache : List.of(NoCache.getInstance(), new CHMCache(), new CHMCache(0))) { + var decoder = stringPointerDecoder(payload, chunkSize, cache); + assertEquals(expected, decoder.decode(0, String.class)); + assertEquals(expected, decoder.decode(0, String.class)); + assertEquals("a", decoder.decode(payload.length + 3, String.class)); + } + } + } + + @Test + public void testMalformedUtf8IsRejectedAcrossChunks() throws IOException { + var payloads = List.of( + new byte[] {(byte) 0x80}, + new byte[] {(byte) 0xC0, (byte) 0xAF}, + new byte[] {(byte) 0xC2}, + new byte[] {(byte) 0xE2, (byte) 0x82}, + new byte[] {(byte) 0xED, (byte) 0xA0, (byte) 0x80}, + new byte[] {(byte) 0xF0, (byte) 0x9F, (byte) 0x92}, + new byte[] {(byte) 0xF4, (byte) 0x90, (byte) 0x80, (byte) 0x80}, + new byte[] {(byte) 0xFF}, + new byte[] {(byte) 0xEF, (byte) 0xBF, (byte) 0xBD, (byte) 0xFF} + ); + for (var payload : payloads) { + for (int chunkSize : new int[] {1, 2, 3, 4, 5, 64}) { + for (var cache : List.of(NoCache.getInstance(), new CHMCache(), new CHMCache(0))) { + var decoder = stringPointerDecoder(payload, chunkSize, cache); + var error = assertThrows( + InvalidDatabaseException.class, + () -> decoder.decode(0, String.class) + ); + assertInstanceOf(CharacterCodingException.class, error.getCause()); + assertEquals("a", decoder.decode(payload.length + 3, String.class)); + assertThrows(InvalidDatabaseException.class, () -> decoder.decode(0, String.class)); + } + } + } + } + + private static Decoder stringPointerDecoder(byte[] payload, int chunkSize, NodeCache cache) { + var data = new byte[payload.length + 5]; + data[0] = 0x20; + data[1] = 2; + data[2] = (byte) (0x40 | payload.length); + System.arraycopy(payload, 0, data, 3, payload.length); + data[data.length - 2] = 0x41; + data[data.length - 1] = 'a'; + if (chunkSize >= data.length) { + return new Decoder(cache, SingleBuffer.wrap(data), 0); + } + var chunks = new ByteBuffer[(data.length + chunkSize - 1) / chunkSize]; + for (int i = 0; i < chunks.length; i++) { + int offset = i * chunkSize; + int size = Math.min(chunkSize, data.length - offset); + chunks[i] = ByteBuffer.wrap(data, offset, size).slice(); + } + return new Decoder(cache, new MultiBuffer(chunks, chunkSize), 0); + } + @Test public void testBooleans() throws IOException { DecoderTest.testTypeDecoding(Type.BOOLEAN, @@ -408,6 +589,823 @@ public void testInvalidControlByte() { containsString("The MaxMind DB file's data section contains bad data")); } + private static void writePointer1(ByteArrayOutputStream out, int target) { + // One-byte-payload pointer (type 1, pointer_size 1) with base 0. + out.write((1 << 5) | ((target >> 8) & 0x7)); + out.write(target & 0xFF); + } + + private static void writePointer(ByteArrayOutputStream out, int target) { + if (target < 1 << 11) { + writePointer1(out, target); + return; + } + + var packed = target - (1 << 11); + out.write((1 << 5) | (1 << 3) | ((packed >> 16) & 0x7)); + out.write((packed >> 8) & 0xFF); + out.write(packed & 0xFF); + } + + private static void writeArrayHeader(ByteArrayOutputStream out, int size) { + if (size < 29) { + out.write(size); + out.write(0x04); + return; + } + if (size < 285) { + out.write(29); + out.write(0x04); + out.write(size - 29); + return; + } + + var encoded = size - 285; + out.write(30); + out.write(0x04); + out.write((encoded >> 8) & 0xFF); + out.write(encoded & 0xFF); + } + + private static void writeMapHeader(ByteArrayOutputStream out, int size) { + if (size <= 28) { + out.write(0xE0 | size); + return; + } + if (size <= 284) { + out.write(0xFD); + out.write(size - 29); + return; + } + out.write(0xFE); + var encoded = size - 285; + out.write((encoded >>> 8) & 0xFF); + out.write(encoded & 0xFF); + } + + private static byte[] unknownFieldWithFlatArray(int size) { + var out = new ByteArrayOutputStream(); + out.write(0xE1); // map with one key/value pair + out.write(0x47); // seven-byte UTF-8 string + out.writeBytes("unknown".getBytes(StandardCharsets.UTF_8)); + writeArrayHeader(out, size); + for (var i = 0; i < size; i++) { + out.write(0xA0); // uint16 with value 0 + } + return out.toByteArray(); + } + + private static byte[] unknownFieldWithFlatMap(int size) { + var out = new ByteArrayOutputStream(); + out.write(0xE1); // map with one key/value pair + out.write(0x47); // seven-byte UTF-8 string + out.writeBytes("unknown".getBytes(StandardCharsets.UTF_8)); + writeMapHeader(out, size); + for (var i = 0; i < size; i++) { + out.write(0x40); // empty UTF-8 string key + out.write(0xA0); // uint16 with value 0 + } + return out.toByteArray(); + } + + private static byte[] nestedArrays(int depth) { + var out = new ByteArrayOutputStream(); + for (var i = 0; i < depth; i++) { + out.write(0x01); // extended type, one element + out.write(0x04); // array + } + out.write(0xA0); // uint16 with value 0 + return out.toByteArray(); + } + + private static byte[] nestedMaps(int depth) { + var out = new ByteArrayOutputStream(); + for (var i = 0; i < depth; i++) { + out.write(0xE1); // map with one key/value pair + out.write(0x40); // empty UTF-8 string key + } + out.write(0xA0); // uint16 with value 0 + return out.toByteArray(); + } + + private record EncodedValue(byte[] data, int offset) { + } + + private static EncodedValue pointerNestedArrays(int depth) { + var out = new ByteArrayOutputStream(); + out.write(0xA0); // uint16 with value 0 + var previous = 0; + for (var i = 0; i < depth; i++) { + var offset = out.size(); + out.write(0x01); // extended type, one element + out.write(0x04); // array + writePointer(out, previous); + previous = offset; + } + return new EncodedValue(out.toByteArray(), previous); + } + + private static EncodedValue pointerNestedMaps(int depth) { + var out = new ByteArrayOutputStream(); + out.write(0xA0); // uint16 with value 0 + var previous = 0; + for (var i = 0; i < depth; i++) { + var offset = out.size(); + out.write(0xE1); // map with one key/value pair + out.write(0x40); // empty UTF-8 string key + writePointer(out, previous); + previous = offset; + } + return new EncodedValue(out.toByteArray(), previous); + } + + private static byte[] inlineArray(int size) { + var out = new ByteArrayOutputStream(); + writeArrayHeader(out, size); + for (var i = 0; i < size; i++) { + out.write(0xA0); // uint16 with value 0 + } + return out.toByteArray(); + } + + @Test + public void testPointerFanOutIsBounded() throws IOException { + // A data section of nested arrays, each holding two pointers to the + // node below, would cost 2**depth decode operations. The decoder bounds + // the number of values it decodes per lookup and rejects the database. + var depth = 100; + var out = new ByteArrayOutputStream(); + out.write(0xA0); // leaf: uint16 with value 0 + var prev = 0; + for (var i = 0; i < depth; i++) { + var offset = out.size(); + out.write(0x02); + out.write(0x04); + writePointer1(out, prev); + writePointer1(out, prev); + prev = offset; + } + + var data = out.toByteArray(); + var top = prev; + for (var cache : List.of(NoCache.getInstance(), new CHMCache())) { + var decoder = new Decoder(cache, SingleBuffer.wrap(data), 0); + assertThrows( + InvalidDatabaseException.class, + () -> decoder.decode(top, Object.class)); + } + } + + @Test + public void testPointerFreeContainerDepthIsBounded() throws IOException { + var atLimit = new Decoder(NoCache.getInstance(), + SingleBuffer.wrap(nestedArrays(TEST_MAX_DEPTH)), 0); + atLimit.decode(0, Object.class); + + var overLimit = new Decoder(NoCache.getInstance(), + SingleBuffer.wrap(nestedArrays(TEST_MAX_DEPTH + 1)), 0); + var ex = assertThrows( + InvalidDatabaseException.class, + () -> overLimit.decode(0, Object.class)); + assertThat(ex.getMessage(), containsString("exceeds the maximum depth")); + } + + @Test + public void testPointerBackedContainerDepthIsBounded() throws IOException { + var atLimit = pointerNestedArrays(TEST_MAX_DEPTH); + var decoderAtLimit = new Decoder(NoCache.getInstance(), + SingleBuffer.wrap(atLimit.data()), 0); + decoderAtLimit.decode(atLimit.offset(), Object.class); + + var overLimit = pointerNestedArrays(TEST_MAX_DEPTH + 1); + var decoderOverLimit = new Decoder(NoCache.getInstance(), + SingleBuffer.wrap(overLimit.data()), 0); + var ex = assertThrows( + InvalidDatabaseException.class, + () -> decoderOverLimit.decode(overLimit.offset(), Object.class)); + assertThat(ex.getMessage(), containsString("exceeds the maximum depth")); + } + + @Test + public void testCachedPointerTargetDepthIsBounded() throws IOException { + var nested = pointerNestedArrays(TEST_MAX_DEPTH); + var out = new ByteArrayOutputStream(); + out.writeBytes(nested.data()); + + var seedPointerOffset = out.size(); + writePointer(out, nested.offset()); + + var outerArrayOffset = out.size(); + out.write(0x01); // extended type, one element + out.write(0x04); // array + writePointer(out, nested.offset()); + + var decoder = new Decoder(new CHMCache(), SingleBuffer.wrap(out.toByteArray()), 0); + decoder.decode(seedPointerOffset, Object.class); + + var ex = assertThrows( + InvalidDatabaseException.class, + () -> decoder.decode(outerArrayOffset, Object.class) + ); + assertThat(ex.getMessage(), containsString("exceeds the maximum depth")); + } + + @Test + public void testContainerDepthFitsReducedThreadStack() throws Exception { + runProbe("-Xss512k", StackProbe.class); + } + + private static void runProbe(String vmArgument, Class probe) throws Exception { + var executable = System.getProperty("os.name").startsWith("Windows") + ? "java.exe" + : "java"; + var java = Path.of(System.getProperty("java.home"), "bin", executable).toString(); + var classPath = System.getProperty( + "surefire.test.class.path", + System.getProperty("java.class.path") + ); + var modulePath = System.getProperty("jdk.module.path"); + if (modulePath != null && !modulePath.isBlank()) { + classPath = String.join(System.getProperty("path.separator"), classPath, modulePath); + } + var process = new ProcessBuilder( + java, + vmArgument, + "-cp", + classPath, + probe.getName() + ).redirectErrorStream(true).start(); + + if (!process.waitFor(15, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new AssertionError(probe.getSimpleName() + " did not finish within 15 seconds"); + } + var output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + assertEquals(0, process.exitValue(), output); + } + + @Test + public void testJavaValueCountBoundary() throws IOException { + var atLimit = new Decoder(NoCache.getInstance(), + SingleBuffer.wrap(inlineArray(65_535)), 0); + var result = (List) atLimit.decode(0, Object.class); + assertEquals(65_535, result.size()); + + var overLimit = new Decoder(NoCache.getInstance(), + SingleBuffer.wrap(inlineArray(65_536)), 0); + var ex = assertThrows( + InvalidDatabaseException.class, + () -> overLimit.decode(0, Object.class)); + assertThat(ex.getMessage(), containsString("exceeds the maximum number of values")); + } + + @Test + public void testUnknownFieldValueCountIsBounded() { + var out = new ByteArrayOutputStream(); + out.write(0xE1); // map with one key/value pair + out.write(0x47); // seven-byte UTF-8 string + out.writeBytes("unknown".getBytes(StandardCharsets.UTF_8)); + out.write(0x1E); // extended type, two-byte size + out.write(0x04); // array + out.write(0xFE); // size = 65,535 + out.write(0xE2); + for (var i = 0; i < 65_535; i++) { + out.write(0xA0); // uint16 with value 0 + } + + var decoder = new Decoder(NoCache.getInstance(), + SingleBuffer.wrap(out.toByteArray()), 0); + var ex = assertThrows( + InvalidDatabaseException.class, + () -> decoder.decode(0, EmptyModel.class)); + assertThat(ex.getMessage(), containsString("exceeds the maximum number of values")); + } + + @Test + public void testUnknownFieldDepthIsBounded() { + var value = nestedArrays(TEST_MAX_DEPTH); + var out = new ByteArrayOutputStream(); + out.write(0xE1); // map with one key/value pair + out.write(0x47); // seven-byte UTF-8 string + out.writeBytes("unknown".getBytes(StandardCharsets.UTF_8)); + out.writeBytes(value); + + var decoder = new Decoder(NoCache.getInstance(), + SingleBuffer.wrap(out.toByteArray()), 0); + var ex = assertThrows( + InvalidDatabaseException.class, + () -> decoder.decode(0, EmptyModel.class)); + assertThat(ex.getMessage(), containsString("exceeds the maximum depth")); + } + + @Test + public void testUnknownFieldPointersAreSkippedByTheirEncodedWidth() throws IOException { + var pointerEncodings = new ArrayList(); + pointerEncodings.add(new byte[] {0x20, 0x00}); + pointerEncodings.add(new byte[] {0x28, 0x00, 0x00}); + pointerEncodings.add(new byte[] {0x30, 0x00, 0x00, 0x00}); + for (var controlByte = 0x38; controlByte <= 0x3F; controlByte++) { + pointerEncodings.add(new byte[] { + (byte) controlByte, 0x00, 0x00, 0x00, 0x00 + }); + } + + for (var pointer : pointerEncodings) { + var out = new ByteArrayOutputStream(); + out.write(0xE2); // map with two key/value pairs + out.write(0x47); // seven-byte UTF-8 string + out.writeBytes("unknown".getBytes(StandardCharsets.UTF_8)); + out.writeBytes(pointer); + out.write(0x45); // five-byte UTF-8 string + out.writeBytes("known".getBytes(StandardCharsets.UTF_8)); + out.write(0x42); // two-byte UTF-8 string + out.writeBytes("ok".getBytes(StandardCharsets.UTF_8)); + + var decoder = new Decoder( + NoCache.getInstance(), + SingleBuffer.wrap(out.toByteArray()), + 0 + ); + var result = decoder.decode(0, KnownFieldModel.class); + assertEquals("ok", result.known()); + } + } + + @Test + public void testTruncatedUnknownPointersAreRejectedAsInvalidDatabase() { + for (var pointerSize = 1; pointerSize <= 4; pointerSize++) { + var out = new ByteArrayOutputStream(); + out.write(0xE1); // map with one key/value pair + out.write(0x47); // seven-byte UTF-8 string + out.writeBytes("unknown".getBytes(StandardCharsets.UTF_8)); + out.write(0x20 | ((pointerSize - 1) << 3)); + out.writeBytes(new byte[pointerSize - 1]); + + var decoder = new Decoder( + NoCache.getInstance(), + SingleBuffer.wrap(out.toByteArray()), + 0 + ); + var ex = assertThrows( + InvalidDatabaseException.class, + () -> decoder.decode(0, EmptyModel.class) + ); + assertThat(ex.getMessage(), containsString("extends beyond the end")); + } + } + + @Test + public void testTruncatedUnknownScalarIsRejectedAsInvalidDatabase() { + var out = new ByteArrayOutputStream(); + out.write(0xE1); // map with one key/value pair + out.write(0x47); // seven-byte UTF-8 string + out.writeBytes("unknown".getBytes(StandardCharsets.UTF_8)); + out.write(0x41); // one-byte UTF-8 string with no payload + + var decoder = new Decoder( + NoCache.getInstance(), + SingleBuffer.wrap(out.toByteArray()), + 0 + ); + var ex = assertThrows( + InvalidDatabaseException.class, + () -> decoder.decode(0, EmptyModel.class) + ); + assertThat(ex.getMessage(), containsString("extends beyond the end")); + } + + @Test + public void testHugeContainerIsRejectedBeforeAllocation() throws IOException { + // An array control byte can declare up to ~16.8 million entries from a + // few bytes. The value limit must reject this before the decoder uses + // the declared size as an allocation hint. + var out = new ByteArrayOutputStream(); + out.write(0x1F); // extended type, size code 31 (three size bytes) + out.write(0x04); // array + out.write(0xFF); // size = 65821 + 0xFFFFFF = 16,843,036 + out.write(0xFF); + out.write(0xFF); + + var decoder = new Decoder(NoCache.getInstance(), SingleBuffer.wrap(out.toByteArray()), 0); + var ex = assertThrows( + InvalidDatabaseException.class, + () -> decoder.decode(0, Object.class)); + assertThat(ex.getMessage(), containsString("exceeds the maximum number of values")); + } + + @Test + public void testArrayInitialCapacityIsBounded() throws IOException { + var decoder = new Decoder(NoCache.getInstance(), + SingleBuffer.wrap(inlineArray(129)), 0); + var result = decoder.decode(0, CapacityList.class); + assertEquals(128, result.initialCapacity); + assertEquals(129, result.size()); + } + + @Test + public void testMapInitialCapacityIsBounded() throws IOException { + var out = new ByteArrayOutputStream(); + out.write(0xFD); // map, size code 29 + out.write(100); // 29 + 100 = 129 entries + for (var i = 0; i < 129; i++) { + var key = Integer.toString(i).getBytes(StandardCharsets.UTF_8); + out.write(0x40 | key.length); + out.writeBytes(key); + out.write(0xA0); // uint16 with value 0 + } + + var decoder = new Decoder(NoCache.getInstance(), + SingleBuffer.wrap(out.toByteArray()), 0); + var result = decoder.decode(0, CapacityMap.class); + assertEquals(128, result.initialCapacity); + assertEquals(129, result.size()); + } + + @Test + public void testNestedLargeCollectionsDoNotExhaustHeap() throws Exception { + runProbe("-Xmx16m", AllocationProbe.class); + } + + @Test + public void testImpossibleArrayIsRejectedBeforeAllocation() { + // The declared size is below the value budget, but two elements cannot + // be encoded in the one remaining byte. + var decoder = new Decoder(NoCache.getInstance(), + SingleBuffer.wrap(new byte[] {0x02, 0x04, (byte) 0xA0}), 0); + var ex = assertThrows( + InvalidDatabaseException.class, + () -> decoder.decode(0, Object.class)); + assertThat(ex.getMessage(), containsString( + "a container declares more entries than the data section can hold")); + } + + @Test + public void testImpossibleMapIsRejectedBeforeAllocation() { + // A one-entry map needs both a key and a value, but only one byte + // remains after its control byte. + var decoder = new Decoder(NoCache.getInstance(), + SingleBuffer.wrap(new byte[] {(byte) 0xE1, (byte) 0xA0}), 0); + var ex = assertThrows( + InvalidDatabaseException.class, + () -> decoder.decode(0, Object.class)); + assertThat(ex.getMessage(), containsString( + "a container declares more entries than the data section can hold")); + } + + @Test + public void testCyclicPointerThrows() { + // A pointer to itself must throw a catchable InvalidDatabaseException + // rather than recursing until the stack overflows. + var decoder = new Decoder(NoCache.getInstance(), + SingleBuffer.wrap(new byte[] {0x20, 0x00}), 0); + assertThrows( + InvalidDatabaseException.class, + () -> decoder.decode(0, Object.class)); + } + + @Test + public void testAcyclicPointerToPointerThrows() { + // The pointer chain terminates at a scalar, but pointer-to-pointer is + // illegal regardless of whether the chain forms a cycle. + var data = new byte[] {0x20, 0x02, 0x20, 0x04, (byte) 0xA0}; + for (var cache : List.of(NoCache.getInstance(), new CHMCache())) { + var decoder = new Decoder(cache, SingleBuffer.wrap(data), 0); + var ex = assertThrows( + InvalidDatabaseException.class, + () -> decoder.decode(0, Object.class)); + assertThat(ex.getMessage(), containsString("pointer to a pointer")); + } + } + + // Writes a large scalar (bytes or string) at offset 0, followed by an array + // of pointerCount one-byte pointers that all target it. NoCache re-decodes + // the shared value for every pointer. CHMCache materializes it once and + // replays its recorded cost. Both paths charge its size once per pointer. + private static byte[] sharedScalarFanOut(int scalarType, int scalarSize, int pointerCount) { + var out = new ByteArrayOutputStream(); + // Scalar header: size code 30 covers 285..65820 bytes. + out.write((scalarType << 5) | 30); + var encoded = scalarSize - 285; + out.write((encoded >> 8) & 0xFF); + out.write(encoded & 0xFF); + for (var i = 0; i < scalarSize; i++) { + out.write(0); + } + // Array header (extended type 11), size code 29 covers 29..284 entries. + out.write(29); + out.write(0x04); + out.write(pointerCount - 29); + for (var i = 0; i < pointerCount; i++) { + writePointer1(out, 0); + } + return out.toByteArray(); + } + + @Test + public void testPayloadAmplificationIsBounded() throws IOException { + // 33 pointers to a 65,536-byte value would materialize just over 2 MiB, + // one byte value at a time, while the value count stays tiny. Only the + // payload byte bound rejects this. + var scalarSize = 1 << 16; + var data = sharedScalarFanOut(4, scalarSize, 33); + var top = 3 + scalarSize; + for (var cache : List.of(NoCache.getInstance(), new CHMCache())) { + var decoder = new Decoder(cache, SingleBuffer.wrap(data), 0); + var ex = assertThrows( + InvalidDatabaseException.class, + () -> decoder.decode(top, Object.class)); + assertThat(ex.getMessage(), containsString("exceeds the maximum payload size")); + } + } + + @Test + public void testPayloadAmplificationIsBoundedAfterCacheFills() { + var scalarSize = 1 << 16; + var data = sharedScalarFanOut(4, scalarSize, 33); + var top = 3 + scalarSize; + var decoder = new Decoder(new CHMCache(0), SingleBuffer.wrap(data), 0); + var ex = assertThrows( + InvalidDatabaseException.class, + () -> decoder.decode(top, Object.class) + ); + assertThat(ex.getMessage(), containsString("exceeds the maximum payload size")); + } + + @Test + public void testOverBudgetPayloadHeadersAreRejectedBeforePayloadRead() { + var overBudgetHeaders = List.of( + new byte[] {0x5F, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF}, + new byte[] {(byte) 0x9F, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF} + ); + for (var header : overBudgetHeaders) { + var decoder = new Decoder(NoCache.getInstance(), SingleBuffer.wrap(header), 0); + var ex = assertThrows( + InvalidDatabaseException.class, + () -> decoder.decode(0, Object.class) + ); + assertThat(ex.getMessage(), containsString("exceeds the maximum payload size")); + } + } + + @Test + public void testTruncatedPayloadsAreRejectedAsInvalidDatabase() { + var headers = List.of( + new byte[] {0x41}, + new byte[] {(byte) 0x81} + ); + for (var header : headers) { + var decoder = new Decoder(NoCache.getInstance(), SingleBuffer.wrap(header), 0); + var ex = assertThrows( + InvalidDatabaseException.class, + () -> decoder.decode(0, Object.class) + ); + assertThat(ex.getMessage(), containsString("extends beyond the end")); + } + } + + @Test + public void testInvalidStringDoesNotChangeBufferLimit() throws IOException { + var data = new byte[] {0x41, (byte) 0xFF, 0x41, 'a'}; + var decoder = new Decoder(NoCache.getInstance(), SingleBuffer.wrap(data), 0); + + var ex = assertThrows( + InvalidDatabaseException.class, + () -> decoder.decode(0, Object.class) + ); + assertThat(ex.getMessage(), containsString("invalid UTF-8 string")); + assertEquals("a", decoder.decode(2, Object.class)); + } + + @Test + public void testSkippedPayloadDoesNotConsumeMaterializationBudget() throws IOException { + var payloadSize = (1 << 21) + 1; + var out = new ByteArrayOutputStream(); + out.write(0xE2); // map with two key/value pairs + out.write(0x47); // seven-byte UTF-8 string + out.writeBytes("unknown".getBytes(StandardCharsets.UTF_8)); + out.write(0x5F); // UTF-8 string, size code 31 + var encodedSize = payloadSize - 65_821; + out.write((encodedSize >>> 16) & 0xFF); + out.write((encodedSize >>> 8) & 0xFF); + out.write(encodedSize & 0xFF); + out.writeBytes(new byte[payloadSize]); + out.write(0x45); // five-byte UTF-8 string + out.writeBytes("known".getBytes(StandardCharsets.UTF_8)); + out.write(0x42); // two-byte UTF-8 string + out.writeBytes("ok".getBytes(StandardCharsets.UTF_8)); + + var decoder = new Decoder( + NoCache.getInstance(), + SingleBuffer.wrap(out.toByteArray()), + 0 + ); + var result = decoder.decode(0, KnownFieldModel.class); + assertEquals("ok", result.known()); + } + + @Test + public void testPayloadAtLimitIsAccepted() throws IOException { + // 32 pointers to a 65,536-byte value materialize exactly 2 MiB, at the + // inclusive limit, so the record must still decode. + var scalarSize = 1 << 16; + var data = sharedScalarFanOut(4, scalarSize, 32); + var top = 3 + scalarSize; + var decoder = new Decoder(NoCache.getInstance(), SingleBuffer.wrap(data), 0); + var result = (List) decoder.decode(top, Object.class); + assertEquals(32, result.size()); + } + + @Test + public void testBigIntegerDoesNotConsumeStringAndBytesBudget() throws IOException { + var payloadSize = 1 << 21; + var out = new ByteArrayOutputStream(); + out.write(0x02); // extended type, two elements + out.write(0x04); // array + out.write(0x10); // 16-byte extended value + out.write(0x03); // uint128 + out.writeBytes(new byte[16]); + + out.write(0x9F); // bytes, size code 31 + var encodedSize = payloadSize - 65_821; + out.write((encodedSize >>> 16) & 0xFF); + out.write((encodedSize >>> 8) & 0xFF); + out.write(encodedSize & 0xFF); + out.writeBytes(new byte[payloadSize]); + + var decoder = new Decoder( + NoCache.getInstance(), + SingleBuffer.wrap(out.toByteArray()), + 0 + ); + var result = (List) decoder.decode(0, Object.class); + assertEquals(2, result.size()); + } + + public static final class StackProbe { + private StackProbe() { + } + + public static void main(String[] args) throws IOException { + decode(nestedArrays(TEST_MAX_DEPTH), 0); + decode(nestedMaps(TEST_MAX_DEPTH), 0); + + var pointerArray = pointerNestedArrays(TEST_MAX_DEPTH); + decode(pointerArray.data(), pointerArray.offset()); + for (var cache : caches()) { + decode(pointerArray.data(), pointerArray.offset(), cache); + } + var pointerMap = pointerNestedMaps(TEST_MAX_DEPTH); + decode(pointerMap.data(), pointerMap.offset()); + for (var cache : caches()) { + decode(pointerMap.data(), pointerMap.offset(), cache); + } + + expectDepthRejection(nestedArrays(TEST_MAX_DEPTH + 1), 0); + expectDepthRejection(nestedMaps(TEST_MAX_DEPTH + 1), 0); + + pointerArray = pointerNestedArrays(TEST_MAX_DEPTH + 1); + expectDepthRejection(pointerArray.data(), pointerArray.offset()); + for (var cache : caches()) { + expectDepthRejection(pointerArray.data(), pointerArray.offset(), cache); + } + pointerMap = pointerNestedMaps(TEST_MAX_DEPTH + 1); + expectDepthRejection(pointerMap.data(), pointerMap.offset()); + for (var cache : caches()) { + expectDepthRejection(pointerMap.data(), pointerMap.offset(), cache); + } + + decodeUnknown(unknownFieldWithFlatArray(65_532)); + decodeUnknown(unknownFieldWithFlatMap(32_766)); + } + + private static void decode(byte[] data, int offset) throws IOException { + decode(data, offset, NoCache.getInstance()); + } + + private static void decode(byte[] data, int offset, NodeCache cache) throws IOException { + var decoder = new Decoder(cache, SingleBuffer.wrap(data), 0); + decoder.decode(offset, Object.class); + } + + private static void expectDepthRejection(byte[] data, int offset) throws IOException { + expectDepthRejection(data, offset, NoCache.getInstance()); + } + + private static void expectDepthRejection( + byte[] data, + int offset, + NodeCache cache + ) throws IOException { + try { + decode(data, offset, cache); + throw new AssertionError("over-depth container decoded without rejection"); + } catch (InvalidDatabaseException e) { + if (!e.getMessage().contains("exceeds the maximum depth")) { + throw e; + } + } + } + + private static List caches() { + return List.of(new CHMCache(), new CHMCache(0)); + } + + private static void decodeUnknown(byte[] data) throws IOException { + var decoder = new Decoder(NoCache.getInstance(), SingleBuffer.wrap(data), 0); + decoder.decode(0, EmptyModel.class); + } + } + + public static final class EmptyModel { + @MaxMindDbConstructor + public EmptyModel() { + } + } + + public static final class KnownFieldModel { + private final String known; + + @MaxMindDbConstructor + public KnownFieldModel(@MaxMindDbParameter(name = "known") String known) { + this.known = known; + } + + public String known() { + return this.known; + } + } + + public static final class CapacityList extends ArrayList { + private static final long serialVersionUID = 1L; + private final int initialCapacity; + + public CapacityList(int initialCapacity) { + super(initialCapacity); + this.initialCapacity = initialCapacity; + } + } + + public static final class CapacityMap extends HashMap { + private static final long serialVersionUID = 1L; + private final int initialCapacity; + + public CapacityMap(int initialCapacity) { + super(initialCapacity); + this.initialCapacity = initialCapacity; + } + } + + public static final class AllocationProbe { + private AllocationProbe() { + } + + public static void main(String[] args) throws IOException { + decodeRecursivelyNestedArray(); + decodeRecursivelyNestedMap(); + } + + private static void decodeRecursivelyNestedArray() throws IOException { + var data = new byte[40_000]; + var encodedSize = 32_768 - 285; + data[0] = 0x1E; // extended type, size code 30 + data[1] = 0x04; // array + data[2] = (byte) (encodedSize >> 8); + data[3] = (byte) encodedSize; + data[4] = 0x20; // one-byte pointer to offset 0 + data[5] = 0x00; + + expectDepthRejection(data); + } + + private static void decodeRecursivelyNestedMap() throws IOException { + var data = new byte[40_000]; + var encodedSize = 16_384 - 285; + data[0] = (byte) 0xFE; // map, size code 30 + data[1] = (byte) (encodedSize >> 8); + data[2] = (byte) encodedSize; + data[3] = 0x41; // one-byte UTF-8 string key + data[4] = 'a'; + data[5] = (byte) 0xA0; // uint16 with value 0 + data[6] = 0x40; // empty UTF-8 string key + data[7] = 0x20; // one-byte pointer to offset 0 + data[8] = 0x00; + + expectDepthRejection(data); + } + + private static void expectDepthRejection(byte[] data) throws IOException { + var decoder = new Decoder(NoCache.getInstance(), SingleBuffer.wrap(data), 0); + try { + decoder.decode(0, Object.class); + throw new AssertionError("nested large collection decoded without rejection"); + } catch (InvalidDatabaseException e) { + if (!e.getMessage().contains("exceeds the maximum depth")) { + throw e; + } + } + } + } + private static void testTypeDecoding(Type type, Map tests) throws IOException { var cache = new CHMCache(); diff --git a/src/test/java/com/maxmind/db/MultiBufferTest.java b/src/test/java/com/maxmind/db/MultiBufferTest.java index 43be354f..4bd339ed 100644 --- a/src/test/java/com/maxmind/db/MultiBufferTest.java +++ b/src/test/java/com/maxmind/db/MultiBufferTest.java @@ -370,4 +370,29 @@ public void testDecodeAcrossChunks() throws CharacterCodingException { assertEquals("123456789012345678901234567", result); assertEquals(89, buffer.position()); } + + @Test + public void testDecodeMultibyteCharacterAcrossChunks() throws CharacterCodingException { + var bytes = "a€b".getBytes(StandardCharsets.UTF_8); + var chunks = new ByteBuffer[]{ + ByteBuffer.wrap(new byte[]{bytes[0], bytes[1]}), + ByteBuffer.wrap(new byte[]{bytes[2], bytes[3]}), + ByteBuffer.wrap(new byte[]{bytes[4]}) + }; + var buffer = new MultiBuffer(chunks, 2); + + assertEquals("a€b", buffer.decode(StandardCharsets.UTF_8.newDecoder())); + assertEquals(bytes.length, buffer.position()); + } + + @Test + public void testDecodeRejectsIncompleteCharacter() { + var bytes = new byte[]{'a', (byte) 0xe2}; + var buffer = new MultiBuffer(new ByteBuffer[]{ByteBuffer.wrap(bytes)}, bytes.length); + + assertThrows( + CharacterCodingException.class, + () -> buffer.decode(StandardCharsets.UTF_8.newDecoder()) + ); + } } diff --git a/src/test/java/com/maxmind/db/ReaderTest.java b/src/test/java/com/maxmind/db/ReaderTest.java index 9188677f..d6ef997a 100644 --- a/src/test/java/com/maxmind/db/ReaderTest.java +++ b/src/test/java/com/maxmind/db/ReaderTest.java @@ -2203,6 +2203,161 @@ public void testNullToPrimitiveErrorMessage(int chunkSize) throws IOException { } } + @ParameterizedTest + @MethodSource("chunkSizes") + public void testPointerFanOutIsRejected(int chunkSize) throws IOException { + var fixtures = new String[] { + "MaxMind-DB-test-pointer-decoder-dos.mmdb", + "MaxMind-DB-test-pointer-decoder-dos-ipv6.mmdb", + }; + var addresses = new String[] {"1.1.1.1", "2001:db8::1"}; + for (var i = 0; i < fixtures.length; i++) { + var fixture = fixtures[i]; + try (var reader = new Reader(getFile(fixture), chunkSize)) { + var address = InetAddress.getByName(addresses[i]); + var ex = assertThrows( + InvalidDatabaseException.class, + () -> reader.get(address, Object.class), + fixture + " should be rejected"); + assertThat(ex.getMessage(), containsString("exceeds the maximum number of values")); + } + } + } + + @Test + public void testPointerFanOutIsRejectedForMemoryAndStreamReaders() throws IOException { + var fixture = "MaxMind-DB-test-pointer-decoder-dos.mmdb"; + var address = InetAddress.getByName("1.1.1.1"); + try (var memoryReader = new Reader(getFile(fixture), FileMode.MEMORY, 512)) { + assertThrows( + InvalidDatabaseException.class, + () -> memoryReader.get(address, Object.class)); + } + try (var streamReader = new Reader(getStream(fixture), 512)) { + assertThrows( + InvalidDatabaseException.class, + () -> streamReader.get(address, Object.class)); + } + } + + @Test + public void testPointerFanOutIsRejectedWithCachedTargets() throws IOException { + var fixture = "MaxMind-DB-test-pointer-decoder-dos.mmdb"; + try (var reader = new Reader(getFile(fixture), new CHMCache())) { + var address = InetAddress.getByName("1.1.1.1"); + for (var i = 0; i < 2; i++) { + var ex = assertThrows( + InvalidDatabaseException.class, + () -> reader.get(address, Object.class) + ); + assertThat(ex.getMessage(), containsString("exceeds the maximum number of values")); + } + } + } + + @Test + public void testSharedValueFixturesUseJavaWorkAccounting() throws IOException { + var fixtures = new String[] { + "MaxMind-DB-test-decoder-value-limit.mmdb", + "MaxMind-DB-test-decoder-value-limit-over.mmdb", + "MaxMind-DB-test-decoder-value-limit-pointer-heavy.mmdb", + }; + var address = InetAddress.getByName("1.1.1.1"); + for (var fixture : fixtures) { + for (var cache : List.of(NoCache.getInstance(), new CHMCache())) { + try (var reader = new Reader(getFile(fixture), cache)) { + var ex = assertThrows( + InvalidDatabaseException.class, + () -> reader.get(address, Object.class), + fixture + " should be rejected under Java work accounting"); + assertThat( + ex.getMessage(), + containsString("exceeds the maximum number of values") + ); + } + } + } + } + + public static final class TargetModel { + final String target; + + @MaxMindDbConstructor + public TargetModel(@MaxMindDbParameter(name = "target") String target) { + this.target = target; + } + } + + @Test + public void testPointerBackedMapKeysSharePayloadBudget() throws IOException { + var fixture = "MaxMind-DB-test-decode-path-shared-budget.mmdb"; + var address = InetAddress.getByName("1.1.1.1"); + for (var cache : List.of(NoCache.getInstance(), new CHMCache())) { + try (var reader = new Reader(getFile(fixture), cache)) { + var ex = assertThrows( + InvalidDatabaseException.class, + () -> reader.get(address, TargetModel.class) + ); + assertThat(ex.getMessage(), containsString("exceeds the maximum payload size")); + } + } + } + + // A crafted database can point many data-section pointers at one large + // string or bytes value. The value count stays low, but a decoder that + // copies each pointer's target materializes N times its size. Decoding must + // reject each of these before it exhausts memory. + @Test + public void testPayloadAmplificationIsRejected() throws IOException { + var fixtures = new String[] { + "MaxMind-DB-test-payload-amplification-dos.mmdb", + "MaxMind-DB-test-payload-amplification-dos-string.mmdb", + "MaxMind-DB-test-payload-amplification-dos-worst-case.mmdb", + "MaxMind-DB-test-decoder-payload-limit-over.mmdb", + }; + var ip = InetAddress.getByName("1.1.1.1"); + for (var fixture : fixtures) { + for (var cache : List.of(NoCache.getInstance(), new CHMCache())) { + try (var reader = new Reader(getFile(fixture), cache)) { + var ex = assertThrows( + InvalidDatabaseException.class, + () -> reader.get(ip, Object.class), + fixture + " should be rejected"); + assertThat( + ex.getMessage(), + containsString("exceeds the maximum payload size") + ); + } + } + } + } + + // A payload total that lands exactly on the 2 MiB limit is valid and must + // still decode, so the bound does not reject legitimate data. + @Test + public void testPayloadAtLimitDecodes() throws IOException { + for (var cache : List.of(NoCache.getInstance(), new CHMCache())) { + try (var reader = new Reader( + getFile("MaxMind-DB-test-decoder-payload-limit.mmdb"), cache)) { + var value = reader.get(InetAddress.getByName("1.1.1.1"), Object.class); + assertNotNull(value); + } + } + } + + // Metadata is decoded while the database is opened, so the payload bound must + // cover that path too. This fixture amplifies a string through the metadata. + @Test + public void testMetadataPayloadAmplificationIsRejected() { + for (var cache : List.of(NoCache.getInstance(), new CHMCache())) { + var ex = assertThrows( + InvalidDatabaseException.class, + () -> new Reader(getFile("MaxMind-DB-test-metadata-payload-limit.mmdb"), cache) + ); + assertThat(ex.getMessage(), containsString("exceeds the maximum payload size")); + } + } + static File getFile(String name) { return new File(ReaderTest.class.getResource("/maxmind-db/test-data/" + name).getFile()); } diff --git a/src/test/java/com/maxmind/db/TestDecoder.java b/src/test/java/com/maxmind/db/TestDecoder.java index 99bf9896..bbc74c51 100644 --- a/src/test/java/com/maxmind/db/TestDecoder.java +++ b/src/test/java/com/maxmind/db/TestDecoder.java @@ -10,9 +10,9 @@ final class TestDecoder extends Decoder { } @Override - DecodedValue decodePointer(long pointer, Class cls, Type genericType) { + Object decodePointer(long pointer, Class cls, Type genericType) { // bypass cache - return new DecodedValue(pointer); + return pointer; } } diff --git a/src/test/resources/maxmind-db b/src/test/resources/maxmind-db index e7b00186..363086b7 160000 --- a/src/test/resources/maxmind-db +++ b/src/test/resources/maxmind-db @@ -1 +1 @@ -Subproject commit e7b0018644317ad6f33eb408f4479ccc4ab0e6fd +Subproject commit 363086b7d90650100e91f954937794c6a090c2a0