Skip to content

Optimize small BitMap with long-backed implementation #871

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jt2594838 merged 6 commits into from
Jul 21, 2026
Merged

Optimize small BitMap with long-backed implementation #871

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
061e749
Add BitMapLongImpl
jt2594838 Jul 20, 2026
1f89e60
Refine BitMapLongImpl
jt2594838 Jul 21, 2026
d3b9b33
Add BitMap performance test report
jt2594838 Jul 21, 2026
d56d730
Remove BitMap performance report file
jt2594838 Jul 21, 2026
5752be0
fix review comment
jt2594838 Jul 21, 2026
ae23861
refine memory calculation
jt2594838 Jul 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
257 changes: 62 additions & 195 deletions java/common/src/main/java/org/apache/tsfile/utils/BitMap.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,224 +22,97 @@
import org.apache.tsfile.i18n.Messages;

import java.util.Arrays;
import java.util.Objects;

public class BitMap {
private static final byte[] BIT_UTIL = new byte[] {1, 2, 4, 8, 16, 32, 64, -128};
private static final byte[] UNMARK_BIT_UTIL =
new byte[] {
(byte) 0XFE, // 11111110
(byte) 0XFD, // 11111101
(byte) 0XFB, // 11111011
(byte) 0XF7, // 11110111
(byte) 0XEF, // 11101111
(byte) 0XDF, // 11011111
(byte) 0XBF, // 10111111
(byte) 0X7F // 01111111
};

private byte[] bits;
private int size;

/** Initialize a BitMap with given size. */

public class BitMap implements Accountable {

private BitMapImpl implementation;

/** Initialize an array-backed BitMap with the given size. */
public BitMap(int size) {
this.size = size;
bits = new byte[getSizeOfBytes(size)];
implementation = new BitMapArrayImpl(size);
}

/** Initialize a BitMap with given size and bytes. */
/** Initialize an array-backed BitMap with the given size and bytes. */
public BitMap(int size, byte[] bits) {
this.size = size;
this.bits = bits;
implementation = new BitMapArrayImpl(size, bits);
}

BitMap(BitMapImpl implementation) {
this.implementation = implementation;
}

public byte[] getByteArray() {
return this.bits;
return implementation.getByteArray();
}

public int getSize() {
return this.size;
return implementation.getSize();
}

/** returns the value of the bit with the specified index. */
public boolean isMarked(int position) {
return (bits[position / Byte.SIZE] & BIT_UTIL[position % Byte.SIZE]) != 0;
return implementation.isMarked(position);
}

/** mark as 1 at all positions. */
public void markAll() {
Arrays.fill(bits, (byte) 0XFF);
implementation.markAll();
}

/** mark as 1 at the given bit position. */
public void mark(int position) {
bits[position / Byte.SIZE] |= BIT_UTIL[position % Byte.SIZE];
implementation.mark(position);
}

public void markRange(int startPosition, int length) {
if (length <= 0) {
return;
}

if (startPosition < 0 || startPosition + length > size) {
throw new IndexOutOfBoundsException(
Messages.format(
"error.common.bitmap_start_length_out_of_range", startPosition, length, size));
}

int bitEnd = startPosition + length - 1;
int byte0 = startPosition >>> 3;
int byte1 = bitEnd >>> 3;

if (byte0 == byte1) {
bits[byte0] |= (byte) (((1 << length) - 1) << (startPosition & 7));
return;
}

bits[byte0++] |= (byte) (0xFF << (startPosition & 7));

while (byte0 < byte1) {
bits[byte0++] = (byte) 0xFF;
}

bits[byte1] |= (byte) (0xFF >>> (7 - (bitEnd & 7)));
implementation.markRange(startPosition, length);
}

/** mark as 0 at all positions. */
public void reset() {
Arrays.fill(bits, (byte) 0);
implementation.reset();
}

public void unmark(int position) {
bits[position / Byte.SIZE] &= UNMARK_BIT_UTIL[position % Byte.SIZE];
implementation.unmark(position);
}

public void unmarkRange(int startPosition, int length) {
if (length <= 0) {
return;
}

if (startPosition < 0 || startPosition + length > size) {
throw new IndexOutOfBoundsException(
Messages.format(
"error.common.bitmap_start_length_out_of_range", startPosition, length, size));
}

int bitEnd = startPosition + length - 1;
int byte0 = startPosition >>> 3;
int byte1 = bitEnd >>> 3;

if (byte0 == byte1) {
bits[byte0] &= (byte) ~(((1 << length) - 1) << (startPosition & 7));
return;
}

bits[byte0++] &= (byte) ~(0xFF << (startPosition & 7));

while (byte0 < byte1) {
bits[byte0++] = 0;
}

bits[byte1] &= (byte) (0xFF << ((bitEnd & 7) + 1));
implementation.unmarkRange(startPosition, length);
}

public void merge(BitMap src, int srcStart, int destStart, int len) {
if (len <= 0) return;
if (srcStart < 0 || destStart < 0 || srcStart + len > src.size || destStart + len > this.size) {
throw new IndexOutOfBoundsException();
}

int done = 0;
int dstBit = destStart & 7;
while (done < len) {
int size = Math.min(len - done, 64);
long bits = extractBits(src.bits, srcStart + done, size);
int destStartByte = (destStart + done) >>> 3;
this.bits[destStartByte++] |= (byte) ((bits << dstBit) & 255L);
bits = bits >>> (8 - dstBit);
while (bits > 0L) {
this.bits[destStartByte++] |= (byte) (bits & 255L);
bits = bits >>> 8;
}
done += size;
}
}

private long extractBits(byte[] buf, int off, int len) {
int start = off >>> 3;
int size = 8 - (off & 7);
long val = (buf[start++] & 0xFFL) >>> (off & 7);
while (size < len) {
val |= ((buf[start++] & 0xFFL) << size);
size += 8;
}

return val & (0xffff_ffff_ffff_ffffL >>> (64 - len));
implementation.merge(src.implementation, srcStart, destStart, len);
}

/** whether all bits are zero, i.e., no Null value */
public boolean isAllUnmarked() {
int j;
for (j = 0; j < size / Byte.SIZE; j++) {
if (bits[j] != (byte) 0) {
return false;
}
}
for (j = 0; j < size % Byte.SIZE; j++) {
if ((bits[size / Byte.SIZE] & BIT_UTIL[j]) != 0) {
return false;
}
}
return true;
return implementation.isAllUnmarked();
}

// whether all bits in the range are unmarked
public boolean isAllUnmarked(int rangeSize) {
int j;
for (j = 0; j < rangeSize / Byte.SIZE; j++) {
if (bits[j] != (byte) 0) {
return false;
}
}
int remainingBits = rangeSize % Byte.SIZE;
if (remainingBits > 0) {
byte mask = (byte) (0xFF >> (Byte.SIZE - remainingBits));
if ((bits[rangeSize / Byte.SIZE] & mask) != 0) {
return false;
}
}
return true;
return implementation.isAllUnmarked(rangeSize);
}

/** whether all bits are one, i.e., all are Null */
public boolean isAllMarked() {
int j;
for (j = 0; j < size / Byte.SIZE; j++) {
if (bits[j] != (byte) 0XFF) {
return false;
}
}
for (j = 0; j < size % Byte.SIZE; j++) {
if ((bits[size / Byte.SIZE] & BIT_UTIL[j]) == 0) {
return false;
}
}
return true;
return implementation.isAllMarked();
}

@Override
public String toString() {
StringBuilder res = new StringBuilder();
for (int i = 0; i < size; i++) {
res.append(isMarked(i) ? 1 : 0);
StringBuilder result = new StringBuilder();
for (int i = 0; i < getSize(); i++) {
result.append(isMarked(i) ? 1 : 0);
}
return res.toString();
return result.toString();
}

@Override
public int hashCode() {
int result = Objects.hash(size);
result = 31 * result + Arrays.hashCode(bits);
int result = 31 + getSize();
result = 31 * result + implementation.contentHashCode();
return result;
}

Expand All @@ -248,56 +121,35 @@ public boolean equals(Object obj) {
if (obj == this) {
return true;
}
if (obj == null) {
return false;
}
if (!(obj instanceof BitMap)) {
return false;
}
BitMap other = (BitMap) obj;
return this.size == other.size && Arrays.equals(this.bits, other.bits);
return getSize() == other.getSize() && implementation.contentEquals(other.implementation);
}

public boolean equalsInRange(Object obj, int rangeSize) {
if (obj == this) {
return true;
}
if (obj == null) {
return false;
}
if (!(obj instanceof BitMap)) {
return false;
}
BitMap other = (BitMap) obj;
if (rangeSize > size || rangeSize > other.size) {
if (rangeSize > getSize() || rangeSize > other.getSize()) {
throw new IllegalArgumentException(
Messages.format(
"error.common.bitmap_range_size_exceeds",
rangeSize,
Math.min(this.size, other.size)));
Math.min(getSize(), other.getSize())));
}

int byteSize = rangeSize / Byte.SIZE;
for (int i = 0; i < byteSize; i++) {
if (this.bits[i] != other.bits[i]) {
return false;
}
}
int remainingBits = rangeSize % Byte.SIZE;
if (remainingBits > 0) {
byte mask = (byte) (0xFF >> (Byte.SIZE - remainingBits));
if ((this.bits[byteSize] & mask) != (other.bits[byteSize] & mask)) {
return false;
}
}
return true;
return implementation.contentEqualsInRange(other.implementation, rangeSize);
}

@Override
public BitMap clone() {
byte[] cloneBytes = new byte[this.bits.length];
System.arraycopy(this.bits, 0, cloneBytes, 0, this.bits.length);
return new BitMap(this.size, cloneBytes);
return new BitMap(implementation.copy());
}

/**
Expand All @@ -316,13 +168,14 @@ public BitMap clone() {
* @throws IndexOutOfBoundsException if copying would cause access of data outside bitmap bounds.
*/
public static void copyOfRange(BitMap src, int srcPos, BitMap dest, int destPos, int length) {
if (srcPos + length > src.size) {
if (srcPos + length > src.getSize()) {
throw new IndexOutOfBoundsException(
Messages.format("error.common.bitmap_out_of_src_range", (srcPos + length - 1), src.size));
} else if (destPos + length > dest.size) {
Messages.format(
"error.common.bitmap_out_of_src_range", (srcPos + length - 1), src.getSize()));
} else if (destPos + length > dest.getSize()) {
throw new IndexOutOfBoundsException(
Messages.format(
"error.common.bitmap_out_of_dest_range", (destPos + length - 1), dest.size));
"error.common.bitmap_out_of_dest_range", (destPos + length - 1), dest.getSize()));
}
for (int i = 0; i < length; ++i) {
if (src.isMarked(srcPos + i)) {
Expand Down Expand Up @@ -350,7 +203,7 @@ public static int getSizeOfBytes(int size) {
}

public byte[] getTruncatedByteArray(int size) {
return Arrays.copyOf(this.bits, getSizeOfBytes(size));
return Arrays.copyOf(getByteArray(), getSizeOfBytes(size));
}

public void append(BitMap another, int position, int length) {
Expand All @@ -364,10 +217,24 @@ public void append(BitMap another, int position, int length) {
}

public void extend(int newSize) {
if (size >= newSize) {
return;
}
bits = Arrays.copyOf(bits, getSizeOfBytes(newSize));
size = newSize;
implementation = implementation.extend(newSize);
}

@Override
public long ramBytesUsed() {
return RamUsageEstimator.BIT_MAP_SIZE + implementation.getRetainedSizeInBytes();
}

BitMapImpl getImplementation() {
return implementation;
}

private static BitMapImpl createImplementation(int size) {
return size >= 0 && size <= Long.SIZE ? new BitMapLongImpl(size) : new BitMapArrayImpl(size);
}

/** Initialize a BitMap whose implementation is selected according to the given size. */
public static BitMap createBitMapDynamically(int size) {
return new BitMap(createImplementation(size));
}
}
Loading
Loading