diff --git a/src/UserGuide/develop/DataFrame/TsFileDataFrame.md b/src/UserGuide/develop/DataFrame/TsFileDataFrame.md index 1b90dcd3e..17d42ffd6 100644 --- a/src/UserGuide/develop/DataFrame/TsFileDataFrame.md +++ b/src/UserGuide/develop/DataFrame/TsFileDataFrame.md @@ -64,9 +64,11 @@ In the table below, `df` is a `TsFileDataFrame` instance, created with | Example | Operation | Returns | |---|---|---| -| `TsFileDataFrame(paths)` | Load a file / list of files / directory | `TsFileDataFrame` | +| `TsFileDataFrame(paths, show_progress=True)` | Load a file / list of files / directory. Set `show_progress=False` to silence loading progress on stderr | `TsFileDataFrame` | | `len(df)` | Number of time series | `int` | +| `df.model` | Loaded file model: `"table"` or `"tree"` | `str` | | `df.list_timeseries("weather")` | Series names, optionally filtered by prefix | `List[str]` | +| `df.list_timeseries_metadata("weather")` | Series metadata, optionally filtered by prefix | `pandas.DataFrame` | | `df["weather.Beijing.humidity"]`, `df[0]`, `df[-1]` | One series | `Timeseries` | | `df["city"]` | A metadata column (a tag / `field` / `start_time` / `end_time` / `count`) | `pandas.Series` | | `df[0:3]`, `df[[0, 2, 5]]` | Subset view by integer position: a contiguous range (`0:3`), or the listed positions (`[0, 2, 5]`); positions are the printed `index` column | `TsFileDataFrame` | @@ -150,6 +152,7 @@ from tsfile import TsFileDataFrame df = TsFileDataFrame(["data/weather.tsfile", "data/sensor.tsfile"]) df = TsFileDataFrame("data/") # recursively find every .tsfile under the directory +df = TsFileDataFrame("data/", show_progress=False) print(df) ``` @@ -203,7 +206,21 @@ optionally filtered by a prefix. Calling it with no argument returns all series. ``` To inspect metadata such as start/end time and count, print the DataFrame (or a -subset of it) — see [Displaying a DataFrame](#displaying-a-dataframe). +subset of it) — see [Displaying a DataFrame](#displaying-a-dataframe) — or call +`list_timeseries_metadata()`. + +## Inspecting metadata + +`list_timeseries_metadata(path_prefix="")` returns a pandas DataFrame indexed by +series name. It includes `field`, `start_time`, `end_time`, `count`, and the +device tag columns. For table-model files it also includes the `table` column. + +```python +meta = df.list_timeseries_metadata() +weather_meta = df.list_timeseries_metadata("weather") + +meta[["field", "start_time", "end_time", "count"]].head() +``` ## Selecting series diff --git a/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md b/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md index 0fb0c1e36..e3fdf0b54 100644 --- a/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md +++ b/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md @@ -32,10 +32,12 @@ typedef enum { TS_DATATYPE_FLOAT = 3, TS_DATATYPE_DOUBLE = 4, TS_DATATYPE_TEXT = 5, + TS_DATATYPE_VECTOR = 6, TS_DATATYPE_TIMESTAMP = 8, TS_DATATYPE_DATE = 9, TS_DATATYPE_BLOB = 10, TS_DATATYPE_STRING = 11, + TS_DATATYPE_NULL_TYPE = 254, TS_DATATYPE_INVALID = 255 } TSDataType; @@ -93,11 +95,49 @@ typedef struct result_set_meta_data { TSDataType* data_types; int column_num; } ResultSetMetaData; + +typedef struct arrow_schema ArrowSchema; +typedef struct arrow_array ArrowArray; + +typedef struct TsFileStatisticBase { + bool has_statistic; + TSDataType type; + int32_t row_count; + int64_t start_time; + int64_t end_time; +} TsFileStatisticBase; + +typedef struct TsFileBoolStatistic { TsFileStatisticBase base; double sum; bool first_bool; bool last_bool; } TsFileBoolStatistic; +typedef struct TsFileIntStatistic { TsFileStatisticBase base; double sum; int64_t min_int64; int64_t max_int64; int64_t first_int64; int64_t last_int64; } TsFileIntStatistic; +typedef struct TsFileFloatStatistic { TsFileStatisticBase base; double sum; double min_float64; double max_float64; double first_float64; double last_float64; } TsFileFloatStatistic; +typedef struct TsFileStringStatistic { TsFileStatisticBase base; char* str_min; char* str_max; char* str_first; char* str_last; } TsFileStringStatistic; +typedef struct TsFileTextStatistic { TsFileStatisticBase base; char* str_first; char* str_last; } TsFileTextStatistic; + +typedef union TimeseriesStatisticUnion { + TsFileBoolStatistic bool_s; + TsFileIntStatistic int_s; + TsFileFloatStatistic float_s; + TsFileStringStatistic string_s; + TsFileTextStatistic text_s; +} TimeseriesStatisticUnion; + +typedef struct TimeseriesStatistic { + TimeseriesStatisticUnion u; +} TimeseriesStatistic; + +#define tsfile_statistic_base(s) ((TsFileStatisticBase*)&(s)->u) ``` > `ColumnSchema` does not carry encoding/compression: on write, columns follow the > global defaults (see [Configuration](#configuration-encoding--compression)); on > read, each column is decoded with the file's actual settings. +> The encoding and compression constants listed above are the values accepted by +> the current writer/configuration path. +> +> `TimeseriesStatistic` is a tagged union in `tsfile_cwrapper.h`. Read common +> fields through `tsfile_statistic_base(&statistic)` and then use the active +> typed member (`int_s`, `float_s`, `bool_s`, `string_s`, or `text_s`) according +> to the statistic data type. ## Write Interface @@ -220,9 +260,11 @@ ERRNO tablet_add_timestamp(Tablet tablet, uint32_t row_index, * @param value [in] Null-terminated string. Ownership remains with caller. * @return ERRNO. */ -ERRNO tablet_add_value_by_name_string(Tablet tablet, uint32_t row_index, - const char* column_name, - const char* value); +ERRNO tablet_add_value_by_name_string_with_len(Tablet tablet, + uint32_t row_index, + const char* column_name, + const char* value, + int value_len); // Supports multiple data types ERRNO tablet_add_value_by_name_int32_t(Tablet tablet, uint32_t row_index, @@ -250,9 +292,11 @@ ERRNO tablet_add_value_by_name_bool(Tablet tablet, uint32_t row_index, * * @param value [in] Null-terminated string. Copied internally. */ -ERRNO tablet_add_value_by_index_string(Tablet tablet, uint32_t row_index, - uint32_t column_index, - const char* value); +ERRNO tablet_add_value_by_index_string_with_len(Tablet tablet, + uint32_t row_index, + uint32_t column_index, + const char* value, + int value_len); // Supports multiple data types @@ -333,7 +377,11 @@ Allowed encodings per data type, and the default used when you do not change it: | `FLOAT`, `DOUBLE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ` | `GORILLA` | | `STRING`, `TEXT` | `PLAIN`, `DICTIONARY` | `PLAIN` | -Compression applies to any data type: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, or `LZ4` (default `LZ4`). +The time column uses the global time configuration and accepts `PLAIN`, +`TS_2DIFF`, `GORILLA`, `ZIGZAG`, `RLE`, or `SPRINTZ`. + +Compression applies to any data type: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, +or `LZ4` (default `LZ4`). ```C // e.g. write every column with LZ4 compression @@ -412,6 +460,31 @@ void free_tsfile_result_set(ResultSet* result_set); ``` +### Tree queries + +```C +/** + * @brief Query tree-model data by measurement names within a time range. + */ +ResultSet tsfile_query_table_on_tree(TsFileReader reader, char** columns, + uint32_t column_num, Timestamp start_time, + Timestamp end_time, ERRNO* err_code); + +/** + * @brief Query tree-model data by row with offset/limit. + * + * @param device_ids Array of device identifiers. + * @param measurement_names Array of measurement names. + * @param offset Leading rows to skip (>= 0). + * @param limit Max rows to return; < 0 means unlimited. + */ +ResultSet tsfile_reader_query_tree_by_row(TsFileReader reader, + char** device_ids, int device_ids_len, + char** measurement_names, + int measurement_names_len, int offset, + int limit, ERRNO* err_code); +``` + ### Filtering by tag @@ -436,6 +509,8 @@ typedef enum { TAG_FILTER_GTEQ = 5, // column >= value TAG_FILTER_REGEXP = 6, // column matches the regex value TAG_FILTER_NOT_REGEXP = 7, // column does not match the regex value + TAG_FILTER_IS_NULL = 8, // column is null + TAG_FILTER_IS_NOT_NULL = 9, // column is not null } TagFilterOp; /** @@ -444,7 +519,7 @@ typedef enum { * @param reader [in] Valid TsFileReader handle. * @param table_name [in] Table whose schema defines the TAG columns. * @param column_name [in] Name of the TAG column to filter on. - * @param value [in] Comparison value (TAG columns are STRING). + * @param value [in] Comparison value (ignored for IS NULL / IS NOT NULL). * @param op [in] Comparison operator (TagFilterOp). * @param err_code [out] RET_OK(0) on success, or error code in errno_define_c.h. * @return TagFilterHandle on success; NULL on failure. @@ -465,6 +540,20 @@ TagFilterHandle tsfile_tag_filter_between(TsFileReader reader, const char* lower, const char* upper, bool is_not, ERRNO* err_code); +// Convenience builders for common single-column predicates. +TagFilterHandle tsfile_tag_filter_eq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_neq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_lt(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_lteq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_gt(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_gteq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); + // Combine predicates. AND/OR/NOT take ownership of their children; free the root only. TagFilterHandle tsfile_tag_filter_and(TagFilterHandle left, TagFilterHandle right); TagFilterHandle tsfile_tag_filter_or(TagFilterHandle left, TagFilterHandle right); @@ -522,6 +611,18 @@ ResultSet tsfile_query_table_with_tag_filter( TagFilterHandle tag_filter, int batch_size, ERRNO* err_code); ``` +### Read batch results as Arrow + +Batch query result sets (`batch_size > 0`) can be fetched as Arrow C Data +Interface arrays and schemas. The caller owns the returned Arrow objects and +must call their `release` callbacks when finished. + +```C +ERRNO tsfile_result_set_get_next_tsblock_as_arrow(ResultSet result_set, + ArrowArray* out_array, + ArrowSchema* out_schema); +``` + Example — read `temperature` only for devices whose `region` TAG equals `shanghai`: @@ -670,11 +771,3 @@ TableSchema* tsfile_reader_get_all_table_schemas(TsFileReader reader, */ void free_table_schema(TableSchema schema); ``` - - - - - - - - diff --git a/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md b/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md index 496e48aca..559cb2b7c 100644 --- a/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md +++ b/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md @@ -31,10 +31,14 @@ enum TSDataType : uint8_t { FLOAT = 3, DOUBLE = 4, TEXT = 5, + VECTOR = 6, + UNKNOWN = 7, TIMESTAMP = 8, DATE = 9, BLOB = 10, STRING = 11, + NULL_TYPE = 254, + INVALID_DATATYPE = 255, }; // Value encoding. See the table below for which encodings apply to which types. @@ -42,10 +46,16 @@ enum TSEncoding : uint8_t { PLAIN = 0, DICTIONARY = 1, RLE = 2, + DIFF = 3, TS_2DIFF = 4, + BITMAP = 5, + GORILLA_V1 = 6, + REGULAR = 7, GORILLA = 8, ZIGZAG = 9, + FREQ = 10, SPRINTZ = 12, + INVALID_ENCODING = 255, }; // Compression type. SNAPPY/GZIP/LZO/LZ4 depend on build options; LZ4 is the default. @@ -54,7 +64,11 @@ enum CompressionType : uint8_t { SNAPPY = 1, GZIP = 2, LZO = 3, + SDT = 4, + PAA = 5, + PLA = 6, LZ4 = 7, + INVALID_COMPRESSION = 255, }; // Column role within a table schema. @@ -300,8 +314,14 @@ uint8_t common::get_global_compression(); // Time-column encoding/compression (the data type is fixed to INT64). int common::set_global_time_encoding(uint8_t encoding); int common::set_global_time_compression(uint8_t compression); +uint8_t common::get_global_time_encoding(); +uint8_t common::get_global_time_compression(); ``` +Global compression accepts `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, and `LZ4`. +The codec enum also contains legacy values such as `SDT`, `PAA`, and `PLA`, but +the global compression setter rejects them. + ## Read Interface ### Tsfile Reader ```cpp @@ -339,6 +359,17 @@ class TsFileReader { * @return Returns 0 on success, or a non-zero error code on failure. */ int query(storage::QueryExpression *qe, ResultSet *&ret_qds); + /** + * @brief query the tsfile by the path list, start time and end time. + * This method is used to query the tree model. + * + * @param [in] path_list the full path list + * @param [in] start_time the start time + * @param [in] end_time the end time + * @param [out] result_set the result set + */ + int query(std::vector& path_list, int64_t start_time, + int64_t end_time, ResultSet*& result_set); /** * @brief query the tsfile by the table name, columns names, start time * and end time. @@ -351,7 +382,7 @@ class TsFileReader { */ int query(const std::string &table_name, const std::vector &columns_names, int64_t start_time, - int64_t end_time, ResultSet *&result_set); + int64_t end_time, ResultSet *&result_set, int batch_size = -1); /** * @brief query the tsfile by the table name, columns names, start time @@ -366,7 +397,14 @@ class TsFileReader { */ int query(const std::string& table_name, const std::vector& columns_names, int64_t start_time, - int64_t end_time, ResultSet*& result_set, Filter* tag_filter); + int64_t end_time, ResultSet*& result_set, Filter* tag_filter, + int batch_size = 0); + + /** + * @brief query tree-model time series by row with offset and limit. + */ + int queryByRow(std::vector& path_list, int offset, int limit, + ResultSet*& result_set); /** * @brief query a table by row, with offset/limit pushdown and an optional @@ -386,6 +424,13 @@ class TsFileReader { int limit, ResultSet*& result_set, Filter* tag_filter = nullptr, int batch_size = 0); + /** + * @brief query tree-model data by measurement names within a time range. + */ + int query_table_on_tree(const std::vector& measurement_names, + int64_t start_time, int64_t end_time, + ResultSet*& result_set); + /** * @brief destroy the result set, this method should be called after the * query is finished and result_set @@ -393,6 +438,26 @@ class TsFileReader { * @param qds the result set */ void destroy_query_data_set(ResultSet *qds); + + ResultSet* read_timeseries( + const std::shared_ptr& device_id, + const std::vector& measurement_name); + + std::vector> get_all_devices( + std::string table_name); + + std::vector> get_all_device_ids(); + + std::vector> get_all_devices(); + + int get_timeseries_schema(std::shared_ptr device_id, + std::vector& result); + + DeviceTimeseriesMetadataMap get_timeseries_metadata( + const std::vector>& device_ids); + + DeviceTimeseriesMetadataMap get_timeseries_metadata(); + /** * @brief get the table schema by the table name * @@ -409,6 +474,11 @@ class TsFileReader { std::vector> get_all_table_schemas(); }; ``` + +`DeviceTimeseriesMetadataMap` is the metadata map returned by +`get_timeseries_metadata()`: `std::map, +std::vector>, IDeviceIDComparator>`. + ### ResultSet ```cpp /** @@ -540,6 +610,8 @@ class TagFilterBuilder { Filter* reg_exp(const std::string& columnName, const std::string& value); Filter* not_reg_exp(const std::string& columnName, const std::string& value); + Filter* is_null(const std::string& columnName); + Filter* is_not_null(const std::string& columnName); Filter* between_and(const std::string& columnName, const std::string& lower, const std::string& upper); Filter* not_between_and(const std::string& columnName, @@ -550,4 +622,4 @@ class TagFilterBuilder { static Filter* or_filter(Filter* left, Filter* right); static Filter* not_filter(Filter* filter); }; -``` \ No newline at end of file +``` diff --git a/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md b/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md index bbcde49cf..f44bbc353 100644 --- a/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md +++ b/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md @@ -47,10 +47,16 @@ class TSEncoding(IntEnum): PLAIN = 0 # all types DICTIONARY = 1 # STRING, TEXT RLE = 2 # INT32, INT64, TIMESTAMP, DATE + DIFF = 3 TS_2DIFF = 4 # INT32, INT64, TIMESTAMP, DATE, FLOAT, DOUBLE + BITMAP = 5 + GORILLA_V1 = 6 + REGULAR = 7 GORILLA = 8 # INT32, INT64, TIMESTAMP, DATE, FLOAT, DOUBLE ZIGZAG = 9 # INT32, INT64 + CHIMP = 11 # INT32, INT64, TIMESTAMP, DATE, FLOAT, DOUBLE SPRINTZ = 12 # INT32, INT64, FLOAT, DOUBLE + RLBE = 13 # INT32, INT64, TIMESTAMP, DATE, FLOAT, DOUBLE class Compressor(IntEnum): """ @@ -60,7 +66,12 @@ class Compressor(IntEnum): SNAPPY = 1 GZIP = 2 LZO = 3 + SDT = 4 + PAA = 5 + PLA = 6 LZ4 = 7 + ZSTD = 8 + LZMA2 = 9 class ColumnCategory(IntEnum): """ @@ -103,13 +114,39 @@ class ResultSetMetaData: def __init__(self, column_list: List[str], data_types: List[TSDataType]) +@dataclass(frozen=True) +class DeviceID: + path: Optional[str] + table_name: Optional[str] + segments: tuple[Optional[str], ...] + +@dataclass(frozen=True) +class TimeseriesStatistic: + has_statistic: bool + row_count: int + start_time: int + end_time: int + +@dataclass(frozen=True) +class TimeseriesMetadata: + measurement_name: str + data_type: TSDataType + chunk_meta_count: int + statistic: TimeseriesStatistic + timeline_statistic: TimeseriesStatistic + +@dataclass(frozen=True) +class DeviceTimeseriesMetadataGroup: + table_name: Optional[str] + segments: tuple[Optional[str], ...] + timeseries: list[TimeseriesMetadata] ``` ## Write interface -### TsFileWriter +### TsFileTableWriter ```python class TsFileTableWriter: @@ -143,6 +180,14 @@ class TsFileTableWriter: """ def write_dataframe(self, dataframe: pandas.DataFrame) + """ + Write a pyarrow RecordBatch or Table into the table. The data must include a + time column and columns matching the table schema. + :param data: pyarrow.RecordBatch or pyarrow.Table. + :return: no return value. + """ + def write_arrow_batch(self, data) + """ Flush buffered data to disk. :return: no return value. @@ -162,8 +207,6 @@ class TsFileTableWriter: def __exit__(self, exc_type, exc_val, exc_tb) ``` - - ### Tablet definition ```Python @@ -235,7 +278,13 @@ encodings per data type, and the default used when you do not change it: | `FLOAT`, `DOUBLE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ` | `GORILLA` | | `STRING`, `TEXT` | `PLAIN`, `DICTIONARY` | `PLAIN` | -Compression applies to any data type: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, or `LZ4` (default `LZ4`). +The time column uses the global time configuration and accepts `PLAIN`, +`TS_2DIFF`, `GORILLA`, `ZIGZAG`, `RLE`, or `SPRINTZ`. + +Compression applies to any data type: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, +or `LZ4` (default `LZ4`). The Python enum also exposes values such as `CHIMP`, +`RLBE`, `ZSTD`, and `LZMA2`, but the current writer/config conversion on +`origin/develop` rejects them. ## Read Interface @@ -262,11 +311,39 @@ class TsFileReader: :param column_names: A list of column names to retrieve. :param start_time: The start time of the query range (default: minimum int64 value). :param end_time: The end time of the query range (default: maximum int64 value). + :param tag_filter: Optional tag predicate for table-model TAG columns. + :param batch_size: <= 0 returns rows one by one; > 0 returns blocks of that size. :return: A query result set handler. """ def query_table(self, table_name : str, column_names : List[str], start_time : int = np.iinfo(np.int64).min, - end_time: int = np.iinfo(np.int64).max) -> ResultSet + end_time: int = np.iinfo(np.int64).max, + tag_filter = None, batch_size : int = 0) -> ResultSet + + """ + Execute a time range query on tree-model measurement columns. + + :param column_names: Measurement names to retrieve. + :param start_time: The start time of the query range. + :param end_time: The end time of the query range. + :return: A query result set handler. + """ + def query_table_on_tree(self, column_names : List[str], + start_time : int = np.iinfo(np.int64).min, + end_time : int = np.iinfo(np.int64).max) -> ResultSet + + """ + Execute tree-model query by row with offset/limit. + + :param device_ids: Device identifiers to query. + :param measurement_names: Measurement names to retrieve. + :param offset: Number of leading rows to skip. + :param limit: Maximum number of rows to return; < 0 means unlimited. + :return: A query result set handler. + """ + def query_tree_by_row(self, device_ids : List[str], + measurement_names : List[str], + offset : int = 0, limit : int = -1) -> ResultSet """ Execute a table query by row, with offset/limit pushdown and an optional @@ -287,6 +364,18 @@ class TsFileReader: offset : int = 0, limit : int = -1, tag_filter = None, batch_size : int = 0) -> ResultSet + """ + Execute a tree-model time range query for one device. + + :param device_name: Device identifier. + :param sensor_list: Measurement names to retrieve. + :param start_time: Query start time. + :param end_time: Query end time. + :return: A query result set handler. + """ + def query_timeseries(self, device_name : str, sensor_list : List[str], + start_time : int = 0, end_time : int = 0) -> ResultSet + """ Retrieves the schema of the specified table. @@ -303,6 +392,31 @@ class TsFileReader: """ def get_all_table_schemas(self) ->dict[str, TableSchema] + """ + Retrieves all tree-model timeseries schemas grouped by device. + + :return: A list of DeviceSchema objects. + """ + def get_all_timeseries_schemas(self) -> list[DeviceSchema] + + """ + Retrieves all device identifiers in the file. + + :return: A list of DeviceID(path, table_name, segments). + """ + def get_all_devices(self) -> List[DeviceID] + + """ + Retrieves per-timeseries metadata for all devices, or only the specified + devices. + + :param device_ids: None for all devices, [] for an empty result, or a list + of DeviceID / path-compatible device identifiers. + :return: dict mapping device segment tuples to DeviceTimeseriesMetadataGroup. + """ + def get_timeseries_metadata( + self, device_ids: Optional[List] = None + ) -> Dict[tuple, DeviceTimeseriesMetadataGroup] """ Closes the TsFile reader. If the reader has active result sets, they will be invalidated. @@ -346,6 +460,13 @@ class ResultSet: """ def read_data_frame(self, max_row_num : int = 1024) -> DataFrame + """ + Fetches the next batch result as a pyarrow.Table. Returns None when no more + TsBlock batches are available. This is only valid for result sets created + with batch_size > 0. + """ + def read_arrow_batch(self) -> Optional[pyarrow.Table] + """ Retrieves the value at the specified index from the query result set. @@ -397,6 +518,9 @@ class ResultSet: Closes the result set and releases any associated resources. """ def close(self) + + def __enter__(self) + def __exit__(self, exc_type, exc_val, exc_tb) ``` ### to_dataframe @@ -467,4 +591,4 @@ def to_dataframe(file_path: str, ColumnNotExistError If any specified column does not exist in the table schema. """ -``` \ No newline at end of file +``` diff --git a/src/UserGuide/latest/DataFrame/TsFileDataFrame.md b/src/UserGuide/latest/DataFrame/TsFileDataFrame.md index 1b90dcd3e..17d42ffd6 100644 --- a/src/UserGuide/latest/DataFrame/TsFileDataFrame.md +++ b/src/UserGuide/latest/DataFrame/TsFileDataFrame.md @@ -64,9 +64,11 @@ In the table below, `df` is a `TsFileDataFrame` instance, created with | Example | Operation | Returns | |---|---|---| -| `TsFileDataFrame(paths)` | Load a file / list of files / directory | `TsFileDataFrame` | +| `TsFileDataFrame(paths, show_progress=True)` | Load a file / list of files / directory. Set `show_progress=False` to silence loading progress on stderr | `TsFileDataFrame` | | `len(df)` | Number of time series | `int` | +| `df.model` | Loaded file model: `"table"` or `"tree"` | `str` | | `df.list_timeseries("weather")` | Series names, optionally filtered by prefix | `List[str]` | +| `df.list_timeseries_metadata("weather")` | Series metadata, optionally filtered by prefix | `pandas.DataFrame` | | `df["weather.Beijing.humidity"]`, `df[0]`, `df[-1]` | One series | `Timeseries` | | `df["city"]` | A metadata column (a tag / `field` / `start_time` / `end_time` / `count`) | `pandas.Series` | | `df[0:3]`, `df[[0, 2, 5]]` | Subset view by integer position: a contiguous range (`0:3`), or the listed positions (`[0, 2, 5]`); positions are the printed `index` column | `TsFileDataFrame` | @@ -150,6 +152,7 @@ from tsfile import TsFileDataFrame df = TsFileDataFrame(["data/weather.tsfile", "data/sensor.tsfile"]) df = TsFileDataFrame("data/") # recursively find every .tsfile under the directory +df = TsFileDataFrame("data/", show_progress=False) print(df) ``` @@ -203,7 +206,21 @@ optionally filtered by a prefix. Calling it with no argument returns all series. ``` To inspect metadata such as start/end time and count, print the DataFrame (or a -subset of it) — see [Displaying a DataFrame](#displaying-a-dataframe). +subset of it) — see [Displaying a DataFrame](#displaying-a-dataframe) — or call +`list_timeseries_metadata()`. + +## Inspecting metadata + +`list_timeseries_metadata(path_prefix="")` returns a pandas DataFrame indexed by +series name. It includes `field`, `start_time`, `end_time`, `count`, and the +device tag columns. For table-model files it also includes the `table` column. + +```python +meta = df.list_timeseries_metadata() +weather_meta = df.list_timeseries_metadata("weather") + +meta[["field", "start_time", "end_time", "count"]].head() +``` ## Selecting series diff --git a/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md b/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md index 0fb0c1e36..e3fdf0b54 100644 --- a/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md +++ b/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md @@ -32,10 +32,12 @@ typedef enum { TS_DATATYPE_FLOAT = 3, TS_DATATYPE_DOUBLE = 4, TS_DATATYPE_TEXT = 5, + TS_DATATYPE_VECTOR = 6, TS_DATATYPE_TIMESTAMP = 8, TS_DATATYPE_DATE = 9, TS_DATATYPE_BLOB = 10, TS_DATATYPE_STRING = 11, + TS_DATATYPE_NULL_TYPE = 254, TS_DATATYPE_INVALID = 255 } TSDataType; @@ -93,11 +95,49 @@ typedef struct result_set_meta_data { TSDataType* data_types; int column_num; } ResultSetMetaData; + +typedef struct arrow_schema ArrowSchema; +typedef struct arrow_array ArrowArray; + +typedef struct TsFileStatisticBase { + bool has_statistic; + TSDataType type; + int32_t row_count; + int64_t start_time; + int64_t end_time; +} TsFileStatisticBase; + +typedef struct TsFileBoolStatistic { TsFileStatisticBase base; double sum; bool first_bool; bool last_bool; } TsFileBoolStatistic; +typedef struct TsFileIntStatistic { TsFileStatisticBase base; double sum; int64_t min_int64; int64_t max_int64; int64_t first_int64; int64_t last_int64; } TsFileIntStatistic; +typedef struct TsFileFloatStatistic { TsFileStatisticBase base; double sum; double min_float64; double max_float64; double first_float64; double last_float64; } TsFileFloatStatistic; +typedef struct TsFileStringStatistic { TsFileStatisticBase base; char* str_min; char* str_max; char* str_first; char* str_last; } TsFileStringStatistic; +typedef struct TsFileTextStatistic { TsFileStatisticBase base; char* str_first; char* str_last; } TsFileTextStatistic; + +typedef union TimeseriesStatisticUnion { + TsFileBoolStatistic bool_s; + TsFileIntStatistic int_s; + TsFileFloatStatistic float_s; + TsFileStringStatistic string_s; + TsFileTextStatistic text_s; +} TimeseriesStatisticUnion; + +typedef struct TimeseriesStatistic { + TimeseriesStatisticUnion u; +} TimeseriesStatistic; + +#define tsfile_statistic_base(s) ((TsFileStatisticBase*)&(s)->u) ``` > `ColumnSchema` does not carry encoding/compression: on write, columns follow the > global defaults (see [Configuration](#configuration-encoding--compression)); on > read, each column is decoded with the file's actual settings. +> The encoding and compression constants listed above are the values accepted by +> the current writer/configuration path. +> +> `TimeseriesStatistic` is a tagged union in `tsfile_cwrapper.h`. Read common +> fields through `tsfile_statistic_base(&statistic)` and then use the active +> typed member (`int_s`, `float_s`, `bool_s`, `string_s`, or `text_s`) according +> to the statistic data type. ## Write Interface @@ -220,9 +260,11 @@ ERRNO tablet_add_timestamp(Tablet tablet, uint32_t row_index, * @param value [in] Null-terminated string. Ownership remains with caller. * @return ERRNO. */ -ERRNO tablet_add_value_by_name_string(Tablet tablet, uint32_t row_index, - const char* column_name, - const char* value); +ERRNO tablet_add_value_by_name_string_with_len(Tablet tablet, + uint32_t row_index, + const char* column_name, + const char* value, + int value_len); // Supports multiple data types ERRNO tablet_add_value_by_name_int32_t(Tablet tablet, uint32_t row_index, @@ -250,9 +292,11 @@ ERRNO tablet_add_value_by_name_bool(Tablet tablet, uint32_t row_index, * * @param value [in] Null-terminated string. Copied internally. */ -ERRNO tablet_add_value_by_index_string(Tablet tablet, uint32_t row_index, - uint32_t column_index, - const char* value); +ERRNO tablet_add_value_by_index_string_with_len(Tablet tablet, + uint32_t row_index, + uint32_t column_index, + const char* value, + int value_len); // Supports multiple data types @@ -333,7 +377,11 @@ Allowed encodings per data type, and the default used when you do not change it: | `FLOAT`, `DOUBLE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ` | `GORILLA` | | `STRING`, `TEXT` | `PLAIN`, `DICTIONARY` | `PLAIN` | -Compression applies to any data type: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, or `LZ4` (default `LZ4`). +The time column uses the global time configuration and accepts `PLAIN`, +`TS_2DIFF`, `GORILLA`, `ZIGZAG`, `RLE`, or `SPRINTZ`. + +Compression applies to any data type: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, +or `LZ4` (default `LZ4`). ```C // e.g. write every column with LZ4 compression @@ -412,6 +460,31 @@ void free_tsfile_result_set(ResultSet* result_set); ``` +### Tree queries + +```C +/** + * @brief Query tree-model data by measurement names within a time range. + */ +ResultSet tsfile_query_table_on_tree(TsFileReader reader, char** columns, + uint32_t column_num, Timestamp start_time, + Timestamp end_time, ERRNO* err_code); + +/** + * @brief Query tree-model data by row with offset/limit. + * + * @param device_ids Array of device identifiers. + * @param measurement_names Array of measurement names. + * @param offset Leading rows to skip (>= 0). + * @param limit Max rows to return; < 0 means unlimited. + */ +ResultSet tsfile_reader_query_tree_by_row(TsFileReader reader, + char** device_ids, int device_ids_len, + char** measurement_names, + int measurement_names_len, int offset, + int limit, ERRNO* err_code); +``` + ### Filtering by tag @@ -436,6 +509,8 @@ typedef enum { TAG_FILTER_GTEQ = 5, // column >= value TAG_FILTER_REGEXP = 6, // column matches the regex value TAG_FILTER_NOT_REGEXP = 7, // column does not match the regex value + TAG_FILTER_IS_NULL = 8, // column is null + TAG_FILTER_IS_NOT_NULL = 9, // column is not null } TagFilterOp; /** @@ -444,7 +519,7 @@ typedef enum { * @param reader [in] Valid TsFileReader handle. * @param table_name [in] Table whose schema defines the TAG columns. * @param column_name [in] Name of the TAG column to filter on. - * @param value [in] Comparison value (TAG columns are STRING). + * @param value [in] Comparison value (ignored for IS NULL / IS NOT NULL). * @param op [in] Comparison operator (TagFilterOp). * @param err_code [out] RET_OK(0) on success, or error code in errno_define_c.h. * @return TagFilterHandle on success; NULL on failure. @@ -465,6 +540,20 @@ TagFilterHandle tsfile_tag_filter_between(TsFileReader reader, const char* lower, const char* upper, bool is_not, ERRNO* err_code); +// Convenience builders for common single-column predicates. +TagFilterHandle tsfile_tag_filter_eq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_neq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_lt(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_lteq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_gt(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_gteq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); + // Combine predicates. AND/OR/NOT take ownership of their children; free the root only. TagFilterHandle tsfile_tag_filter_and(TagFilterHandle left, TagFilterHandle right); TagFilterHandle tsfile_tag_filter_or(TagFilterHandle left, TagFilterHandle right); @@ -522,6 +611,18 @@ ResultSet tsfile_query_table_with_tag_filter( TagFilterHandle tag_filter, int batch_size, ERRNO* err_code); ``` +### Read batch results as Arrow + +Batch query result sets (`batch_size > 0`) can be fetched as Arrow C Data +Interface arrays and schemas. The caller owns the returned Arrow objects and +must call their `release` callbacks when finished. + +```C +ERRNO tsfile_result_set_get_next_tsblock_as_arrow(ResultSet result_set, + ArrowArray* out_array, + ArrowSchema* out_schema); +``` + Example — read `temperature` only for devices whose `region` TAG equals `shanghai`: @@ -670,11 +771,3 @@ TableSchema* tsfile_reader_get_all_table_schemas(TsFileReader reader, */ void free_table_schema(TableSchema schema); ``` - - - - - - - - diff --git a/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md b/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md index 496e48aca..559cb2b7c 100644 --- a/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md +++ b/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md @@ -31,10 +31,14 @@ enum TSDataType : uint8_t { FLOAT = 3, DOUBLE = 4, TEXT = 5, + VECTOR = 6, + UNKNOWN = 7, TIMESTAMP = 8, DATE = 9, BLOB = 10, STRING = 11, + NULL_TYPE = 254, + INVALID_DATATYPE = 255, }; // Value encoding. See the table below for which encodings apply to which types. @@ -42,10 +46,16 @@ enum TSEncoding : uint8_t { PLAIN = 0, DICTIONARY = 1, RLE = 2, + DIFF = 3, TS_2DIFF = 4, + BITMAP = 5, + GORILLA_V1 = 6, + REGULAR = 7, GORILLA = 8, ZIGZAG = 9, + FREQ = 10, SPRINTZ = 12, + INVALID_ENCODING = 255, }; // Compression type. SNAPPY/GZIP/LZO/LZ4 depend on build options; LZ4 is the default. @@ -54,7 +64,11 @@ enum CompressionType : uint8_t { SNAPPY = 1, GZIP = 2, LZO = 3, + SDT = 4, + PAA = 5, + PLA = 6, LZ4 = 7, + INVALID_COMPRESSION = 255, }; // Column role within a table schema. @@ -300,8 +314,14 @@ uint8_t common::get_global_compression(); // Time-column encoding/compression (the data type is fixed to INT64). int common::set_global_time_encoding(uint8_t encoding); int common::set_global_time_compression(uint8_t compression); +uint8_t common::get_global_time_encoding(); +uint8_t common::get_global_time_compression(); ``` +Global compression accepts `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, and `LZ4`. +The codec enum also contains legacy values such as `SDT`, `PAA`, and `PLA`, but +the global compression setter rejects them. + ## Read Interface ### Tsfile Reader ```cpp @@ -339,6 +359,17 @@ class TsFileReader { * @return Returns 0 on success, or a non-zero error code on failure. */ int query(storage::QueryExpression *qe, ResultSet *&ret_qds); + /** + * @brief query the tsfile by the path list, start time and end time. + * This method is used to query the tree model. + * + * @param [in] path_list the full path list + * @param [in] start_time the start time + * @param [in] end_time the end time + * @param [out] result_set the result set + */ + int query(std::vector& path_list, int64_t start_time, + int64_t end_time, ResultSet*& result_set); /** * @brief query the tsfile by the table name, columns names, start time * and end time. @@ -351,7 +382,7 @@ class TsFileReader { */ int query(const std::string &table_name, const std::vector &columns_names, int64_t start_time, - int64_t end_time, ResultSet *&result_set); + int64_t end_time, ResultSet *&result_set, int batch_size = -1); /** * @brief query the tsfile by the table name, columns names, start time @@ -366,7 +397,14 @@ class TsFileReader { */ int query(const std::string& table_name, const std::vector& columns_names, int64_t start_time, - int64_t end_time, ResultSet*& result_set, Filter* tag_filter); + int64_t end_time, ResultSet*& result_set, Filter* tag_filter, + int batch_size = 0); + + /** + * @brief query tree-model time series by row with offset and limit. + */ + int queryByRow(std::vector& path_list, int offset, int limit, + ResultSet*& result_set); /** * @brief query a table by row, with offset/limit pushdown and an optional @@ -386,6 +424,13 @@ class TsFileReader { int limit, ResultSet*& result_set, Filter* tag_filter = nullptr, int batch_size = 0); + /** + * @brief query tree-model data by measurement names within a time range. + */ + int query_table_on_tree(const std::vector& measurement_names, + int64_t start_time, int64_t end_time, + ResultSet*& result_set); + /** * @brief destroy the result set, this method should be called after the * query is finished and result_set @@ -393,6 +438,26 @@ class TsFileReader { * @param qds the result set */ void destroy_query_data_set(ResultSet *qds); + + ResultSet* read_timeseries( + const std::shared_ptr& device_id, + const std::vector& measurement_name); + + std::vector> get_all_devices( + std::string table_name); + + std::vector> get_all_device_ids(); + + std::vector> get_all_devices(); + + int get_timeseries_schema(std::shared_ptr device_id, + std::vector& result); + + DeviceTimeseriesMetadataMap get_timeseries_metadata( + const std::vector>& device_ids); + + DeviceTimeseriesMetadataMap get_timeseries_metadata(); + /** * @brief get the table schema by the table name * @@ -409,6 +474,11 @@ class TsFileReader { std::vector> get_all_table_schemas(); }; ``` + +`DeviceTimeseriesMetadataMap` is the metadata map returned by +`get_timeseries_metadata()`: `std::map, +std::vector>, IDeviceIDComparator>`. + ### ResultSet ```cpp /** @@ -540,6 +610,8 @@ class TagFilterBuilder { Filter* reg_exp(const std::string& columnName, const std::string& value); Filter* not_reg_exp(const std::string& columnName, const std::string& value); + Filter* is_null(const std::string& columnName); + Filter* is_not_null(const std::string& columnName); Filter* between_and(const std::string& columnName, const std::string& lower, const std::string& upper); Filter* not_between_and(const std::string& columnName, @@ -550,4 +622,4 @@ class TagFilterBuilder { static Filter* or_filter(Filter* left, Filter* right); static Filter* not_filter(Filter* filter); }; -``` \ No newline at end of file +``` diff --git a/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md b/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md index bbcde49cf..f44bbc353 100644 --- a/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md +++ b/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md @@ -47,10 +47,16 @@ class TSEncoding(IntEnum): PLAIN = 0 # all types DICTIONARY = 1 # STRING, TEXT RLE = 2 # INT32, INT64, TIMESTAMP, DATE + DIFF = 3 TS_2DIFF = 4 # INT32, INT64, TIMESTAMP, DATE, FLOAT, DOUBLE + BITMAP = 5 + GORILLA_V1 = 6 + REGULAR = 7 GORILLA = 8 # INT32, INT64, TIMESTAMP, DATE, FLOAT, DOUBLE ZIGZAG = 9 # INT32, INT64 + CHIMP = 11 # INT32, INT64, TIMESTAMP, DATE, FLOAT, DOUBLE SPRINTZ = 12 # INT32, INT64, FLOAT, DOUBLE + RLBE = 13 # INT32, INT64, TIMESTAMP, DATE, FLOAT, DOUBLE class Compressor(IntEnum): """ @@ -60,7 +66,12 @@ class Compressor(IntEnum): SNAPPY = 1 GZIP = 2 LZO = 3 + SDT = 4 + PAA = 5 + PLA = 6 LZ4 = 7 + ZSTD = 8 + LZMA2 = 9 class ColumnCategory(IntEnum): """ @@ -103,13 +114,39 @@ class ResultSetMetaData: def __init__(self, column_list: List[str], data_types: List[TSDataType]) +@dataclass(frozen=True) +class DeviceID: + path: Optional[str] + table_name: Optional[str] + segments: tuple[Optional[str], ...] + +@dataclass(frozen=True) +class TimeseriesStatistic: + has_statistic: bool + row_count: int + start_time: int + end_time: int + +@dataclass(frozen=True) +class TimeseriesMetadata: + measurement_name: str + data_type: TSDataType + chunk_meta_count: int + statistic: TimeseriesStatistic + timeline_statistic: TimeseriesStatistic + +@dataclass(frozen=True) +class DeviceTimeseriesMetadataGroup: + table_name: Optional[str] + segments: tuple[Optional[str], ...] + timeseries: list[TimeseriesMetadata] ``` ## Write interface -### TsFileWriter +### TsFileTableWriter ```python class TsFileTableWriter: @@ -143,6 +180,14 @@ class TsFileTableWriter: """ def write_dataframe(self, dataframe: pandas.DataFrame) + """ + Write a pyarrow RecordBatch or Table into the table. The data must include a + time column and columns matching the table schema. + :param data: pyarrow.RecordBatch or pyarrow.Table. + :return: no return value. + """ + def write_arrow_batch(self, data) + """ Flush buffered data to disk. :return: no return value. @@ -162,8 +207,6 @@ class TsFileTableWriter: def __exit__(self, exc_type, exc_val, exc_tb) ``` - - ### Tablet definition ```Python @@ -235,7 +278,13 @@ encodings per data type, and the default used when you do not change it: | `FLOAT`, `DOUBLE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ` | `GORILLA` | | `STRING`, `TEXT` | `PLAIN`, `DICTIONARY` | `PLAIN` | -Compression applies to any data type: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, or `LZ4` (default `LZ4`). +The time column uses the global time configuration and accepts `PLAIN`, +`TS_2DIFF`, `GORILLA`, `ZIGZAG`, `RLE`, or `SPRINTZ`. + +Compression applies to any data type: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, +or `LZ4` (default `LZ4`). The Python enum also exposes values such as `CHIMP`, +`RLBE`, `ZSTD`, and `LZMA2`, but the current writer/config conversion on +`origin/develop` rejects them. ## Read Interface @@ -262,11 +311,39 @@ class TsFileReader: :param column_names: A list of column names to retrieve. :param start_time: The start time of the query range (default: minimum int64 value). :param end_time: The end time of the query range (default: maximum int64 value). + :param tag_filter: Optional tag predicate for table-model TAG columns. + :param batch_size: <= 0 returns rows one by one; > 0 returns blocks of that size. :return: A query result set handler. """ def query_table(self, table_name : str, column_names : List[str], start_time : int = np.iinfo(np.int64).min, - end_time: int = np.iinfo(np.int64).max) -> ResultSet + end_time: int = np.iinfo(np.int64).max, + tag_filter = None, batch_size : int = 0) -> ResultSet + + """ + Execute a time range query on tree-model measurement columns. + + :param column_names: Measurement names to retrieve. + :param start_time: The start time of the query range. + :param end_time: The end time of the query range. + :return: A query result set handler. + """ + def query_table_on_tree(self, column_names : List[str], + start_time : int = np.iinfo(np.int64).min, + end_time : int = np.iinfo(np.int64).max) -> ResultSet + + """ + Execute tree-model query by row with offset/limit. + + :param device_ids: Device identifiers to query. + :param measurement_names: Measurement names to retrieve. + :param offset: Number of leading rows to skip. + :param limit: Maximum number of rows to return; < 0 means unlimited. + :return: A query result set handler. + """ + def query_tree_by_row(self, device_ids : List[str], + measurement_names : List[str], + offset : int = 0, limit : int = -1) -> ResultSet """ Execute a table query by row, with offset/limit pushdown and an optional @@ -287,6 +364,18 @@ class TsFileReader: offset : int = 0, limit : int = -1, tag_filter = None, batch_size : int = 0) -> ResultSet + """ + Execute a tree-model time range query for one device. + + :param device_name: Device identifier. + :param sensor_list: Measurement names to retrieve. + :param start_time: Query start time. + :param end_time: Query end time. + :return: A query result set handler. + """ + def query_timeseries(self, device_name : str, sensor_list : List[str], + start_time : int = 0, end_time : int = 0) -> ResultSet + """ Retrieves the schema of the specified table. @@ -303,6 +392,31 @@ class TsFileReader: """ def get_all_table_schemas(self) ->dict[str, TableSchema] + """ + Retrieves all tree-model timeseries schemas grouped by device. + + :return: A list of DeviceSchema objects. + """ + def get_all_timeseries_schemas(self) -> list[DeviceSchema] + + """ + Retrieves all device identifiers in the file. + + :return: A list of DeviceID(path, table_name, segments). + """ + def get_all_devices(self) -> List[DeviceID] + + """ + Retrieves per-timeseries metadata for all devices, or only the specified + devices. + + :param device_ids: None for all devices, [] for an empty result, or a list + of DeviceID / path-compatible device identifiers. + :return: dict mapping device segment tuples to DeviceTimeseriesMetadataGroup. + """ + def get_timeseries_metadata( + self, device_ids: Optional[List] = None + ) -> Dict[tuple, DeviceTimeseriesMetadataGroup] """ Closes the TsFile reader. If the reader has active result sets, they will be invalidated. @@ -346,6 +460,13 @@ class ResultSet: """ def read_data_frame(self, max_row_num : int = 1024) -> DataFrame + """ + Fetches the next batch result as a pyarrow.Table. Returns None when no more + TsBlock batches are available. This is only valid for result sets created + with batch_size > 0. + """ + def read_arrow_batch(self) -> Optional[pyarrow.Table] + """ Retrieves the value at the specified index from the query result set. @@ -397,6 +518,9 @@ class ResultSet: Closes the result set and releases any associated resources. """ def close(self) + + def __enter__(self) + def __exit__(self, exc_type, exc_val, exc_tb) ``` ### to_dataframe @@ -467,4 +591,4 @@ def to_dataframe(file_path: str, ColumnNotExistError If any specified column does not exist in the table schema. """ -``` \ No newline at end of file +``` diff --git a/src/zh/UserGuide/develop/DataFrame/TsFileDataFrame.md b/src/zh/UserGuide/develop/DataFrame/TsFileDataFrame.md index 9fec27c88..263071bc6 100644 --- a/src/zh/UserGuide/develop/DataFrame/TsFileDataFrame.md +++ b/src/zh/UserGuide/develop/DataFrame/TsFileDataFrame.md @@ -59,9 +59,11 @@ data.values # -> np.ndarray, shape (N, 2):N | 示例 | 操作 | 返回类型 | |---|---|---| -| `TsFileDataFrame(paths)` | 加载文件 / 文件列表 / 目录 | `TsFileDataFrame` | +| `TsFileDataFrame(paths, show_progress=True)` | 加载文件 / 文件列表 / 目录。设置 `show_progress=False` 可关闭 stderr 上的加载进度 | `TsFileDataFrame` | | `len(df)` | 时间序列总数 | `int` | +| `df.model` | 已加载文件模型:`"table"` 或 `"tree"` | `str` | | `df.list_timeseries("weather")` | 获取序列名,可按前缀筛选 | `List[str]` | +| `df.list_timeseries_metadata("weather")` | 获取序列元数据,可按前缀筛选 | `pandas.DataFrame` | | `df["weather.Beijing.humidity"]`、`df[0]`、`df[-1]` | 获取单条序列 | `Timeseries` | | `df["city"]` | 获取某元数据列(标签 / `field` / `start_time` / `end_time` / `count`) | `pandas.Series` | | `df[0:3]`、`df[[0, 2, 5]]` | 按整数位置取子集视图:连续区间(`0:3`)或所列位置(`[0, 2, 5]`);位置即打印的 `index` 列 | `TsFileDataFrame` | @@ -132,6 +134,7 @@ from tsfile import TsFileDataFrame df = TsFileDataFrame(["data/weather.tsfile", "data/sensor.tsfile"]) df = TsFileDataFrame("data/") # 递归查找目录下所有 .tsfile +df = TsFileDataFrame("data/", show_progress=False) print(df) ``` @@ -172,7 +175,21 @@ TsFileDataFrame(table model, 972 time series, 5 files) ['weather.Beijing.humidity', 'weather.Beijing.temperature'] ``` -若需查看起止时间、点数等元信息,可打印 DataFrame(或其子集)——见[DataFrame 的展示](#dataframe-的展示)。 +若需查看起止时间、点数等元信息,可打印 DataFrame(或其子集)——见[DataFrame 的展示](#dataframe-的展示)——也可以调用 +`list_timeseries_metadata()`。 + +## 查看元数据 + +`list_timeseries_metadata(path_prefix="")` 返回以序列名为索引的 pandas DataFrame。 +其中包含 `field`、`start_time`、`end_time`、`count` 以及设备标签列。对于 +table 模型文件,还会包含 `table` 列。 + +```python +meta = df.list_timeseries_metadata() +weather_meta = df.list_timeseries_metadata("weather") + +meta[["field", "start_time", "end_time", "count"]].head() +``` ## 选取序列 diff --git a/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md b/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md index de6d39796..9d3b5ea71 100644 --- a/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md +++ b/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md @@ -32,10 +32,12 @@ typedef enum { TS_DATATYPE_FLOAT = 3, TS_DATATYPE_DOUBLE = 4, TS_DATATYPE_TEXT = 5, + TS_DATATYPE_VECTOR = 6, TS_DATATYPE_TIMESTAMP = 8, TS_DATATYPE_DATE = 9, TS_DATATYPE_BLOB = 10, TS_DATATYPE_STRING = 11, + TS_DATATYPE_NULL_TYPE = 254, TS_DATATYPE_INVALID = 255 } TSDataType; @@ -89,9 +91,45 @@ typedef struct result_set_meta_data { TSDataType* data_types; int column_num; } ResultSetMetaData; + +typedef struct arrow_schema ArrowSchema; +typedef struct arrow_array ArrowArray; + +typedef struct TsFileStatisticBase { + bool has_statistic; + TSDataType type; + int32_t row_count; + int64_t start_time; + int64_t end_time; +} TsFileStatisticBase; + +typedef struct TsFileBoolStatistic { TsFileStatisticBase base; double sum; bool first_bool; bool last_bool; } TsFileBoolStatistic; +typedef struct TsFileIntStatistic { TsFileStatisticBase base; double sum; int64_t min_int64; int64_t max_int64; int64_t first_int64; int64_t last_int64; } TsFileIntStatistic; +typedef struct TsFileFloatStatistic { TsFileStatisticBase base; double sum; double min_float64; double max_float64; double first_float64; double last_float64; } TsFileFloatStatistic; +typedef struct TsFileStringStatistic { TsFileStatisticBase base; char* str_min; char* str_max; char* str_first; char* str_last; } TsFileStringStatistic; +typedef struct TsFileTextStatistic { TsFileStatisticBase base; char* str_first; char* str_last; } TsFileTextStatistic; + +typedef union TimeseriesStatisticUnion { + TsFileBoolStatistic bool_s; + TsFileIntStatistic int_s; + TsFileFloatStatistic float_s; + TsFileStringStatistic string_s; + TsFileTextStatistic text_s; +} TimeseriesStatisticUnion; + +typedef struct TimeseriesStatistic { + TimeseriesStatisticUnion u; +} TimeseriesStatistic; + +#define tsfile_statistic_base(s) ((TsFileStatisticBase*)&(s)->u) ``` > `ColumnSchema` 不携带编码/压缩:写入时列遵循全局默认值(见[配置](#配置编码与压缩)),读取时按文件中的实际配置解码。 +> 上面列出的编码和压缩常量是当前 writer/configuration 路径实际接受的值。 +> +> `TimeseriesStatistic` 是 `tsfile_cwrapper.h` 中的带标签 union。可通过 +> `tsfile_statistic_base(&statistic)` 读取通用字段,再根据数据类型读取 +> `int_s`、`float_s`、`bool_s`、`string_s` 或 `text_s`。 ## 写入接口 @@ -207,9 +245,11 @@ ERRNO tablet_add_timestamp(Tablet tablet, uint32_t row_index, * * @param value [输入] 以 '\0' 结尾的字符串,调用者保留所有权。 */ -ERRNO tablet_add_value_by_name_string(Tablet tablet, uint32_t row_index, - const char* column_name, - const char* value); +ERRNO tablet_add_value_by_name_string_with_len(Tablet tablet, + uint32_t row_index, + const char* column_name, + const char* value, + int value_len); // 支持多种数据类型插入 ERRNO tablet_add_value_by_name_int32_t(Tablet tablet, uint32_t row_index, @@ -236,9 +276,11 @@ ERRNO tablet_add_value_by_name_bool(Tablet tablet, uint32_t row_index, * * @param value [输入] 字符串会被内部复制。 */ -ERRNO tablet_add_value_by_index_string(Tablet tablet, uint32_t row_index, - uint32_t column_index, - const char* value); +ERRNO tablet_add_value_by_index_string_with_len(Tablet tablet, + uint32_t row_index, + uint32_t column_index, + const char* value, + int value_len); ERRNO tablet_add_value_by_index_int32_t(Tablet tablet, uint32_t row_index, @@ -314,6 +356,8 @@ uint8_t get_global_time_compression(); | `FLOAT`、`DOUBLE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ` | `GORILLA` | | `STRING`、`TEXT` | `PLAIN`、`DICTIONARY` | `PLAIN` | +时间列使用全局时间配置,支持 `PLAIN`、`TS_2DIFF`、`GORILLA`、`ZIGZAG`、`RLE` 或 `SPRINTZ`。 + 压缩适用于所有数据类型:`UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`(默认 `LZ4`)。 ```C @@ -387,6 +431,31 @@ bool tsfile_result_set_next(ResultSet result_set, ERRNO* error_code); void free_tsfile_result_set(ResultSet* result_set); ``` +### Tree 查询 + +```C +/** + * @brief 按测点名在时间范围内查询 tree 模型数据。 + */ +ResultSet tsfile_query_table_on_tree(TsFileReader reader, char** columns, + uint32_t column_num, Timestamp start_time, + Timestamp end_time, ERRNO* err_code); + +/** + * @brief 按行查询 tree 模型数据,支持 offset/limit。 + * + * @param device_ids 设备标识数组。 + * @param measurement_names 测点名数组。 + * @param offset 需要跳过的起始行数(>= 0)。 + * @param limit 最多返回行数;< 0 表示不限制。 + */ +ResultSet tsfile_reader_query_tree_by_row(TsFileReader reader, + char** device_ids, int device_ids_len, + char** measurement_names, + int measurement_names_len, int offset, + int limit, ERRNO* err_code); +``` + ### 按标签过滤 @@ -409,6 +478,8 @@ typedef enum { TAG_FILTER_GTEQ = 5, // 列 >= 值 TAG_FILTER_REGEXP = 6, // 列匹配正则 值 TAG_FILTER_NOT_REGEXP = 7, // 列不匹配正则 值 + TAG_FILTER_IS_NULL = 8, // 列为空 + TAG_FILTER_IS_NOT_NULL = 9, // 列不为空 } TagFilterOp; /** @@ -417,7 +488,7 @@ typedef enum { * @param reader [输入] 有效的 TsFileReader 句柄。 * @param table_name [输入] 其 schema 定义了这些标签列的表名。 * @param column_name [输入] 要过滤的标签列名。 - * @param value [输入] 比较值(标签列为 STRING 类型)。 + * @param value [输入] 比较值(IS NULL / IS NOT NULL 会忽略该参数)。 * @param op [输入] 比较运算符(TagFilterOp)。 * @param err_code [输出] 成功返回 RET_OK(0),否则返回 errno_define_c.h 中的错误码。 * @return 成功返回 TagFilterHandle,失败返回 NULL。 @@ -437,6 +508,20 @@ TagFilterHandle tsfile_tag_filter_between(TsFileReader reader, const char* lower, const char* upper, bool is_not, ERRNO* err_code); +// 常用单列谓词的便捷构造函数。 +TagFilterHandle tsfile_tag_filter_eq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_neq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_lt(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_lteq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_gt(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_gteq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); + // 组合谓词。AND/OR/NOT 会接管其子节点的所有权,只需释放根节点。 TagFilterHandle tsfile_tag_filter_and(TagFilterHandle left, TagFilterHandle right); TagFilterHandle tsfile_tag_filter_or(TagFilterHandle left, TagFilterHandle right); @@ -493,6 +578,18 @@ ResultSet tsfile_query_table_with_tag_filter( TagFilterHandle tag_filter, int batch_size, ERRNO* err_code); ``` +### 将批结果读取为 Arrow + +对 `batch_size > 0` 创建的批查询结果集,可以使用 Arrow C Data Interface 读取 +`ArrowArray` 和 `ArrowSchema`。调用者拥有返回的 Arrow 对象,使用完后必须调用 +它们的 `release` 回调。 + +```C +ERRNO tsfile_result_set_get_next_tsblock_as_arrow(ResultSet result_set, + ArrowArray* out_array, + ArrowSchema* out_schema); +``` + 示例——只读取 `region` 标签等于 `shanghai` 的设备的 `temperature`: ```C @@ -648,4 +745,3 @@ TableSchema* tsfile_reader_get_all_table_schemas(TsFileReader reader, void free_table_schema(TableSchema schema); ``` - diff --git a/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md b/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md index 50de78b47..5e18d2dac 100644 --- a/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md +++ b/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md @@ -31,10 +31,14 @@ enum TSDataType : uint8_t { FLOAT = 3, DOUBLE = 4, TEXT = 5, + VECTOR = 6, + UNKNOWN = 7, TIMESTAMP = 8, DATE = 9, BLOB = 10, STRING = 11, + NULL_TYPE = 254, + INVALID_DATATYPE = 255, }; // 值编码。各编码适用于哪些类型见下表。 @@ -42,10 +46,16 @@ enum TSEncoding : uint8_t { PLAIN = 0, DICTIONARY = 1, RLE = 2, + DIFF = 3, TS_2DIFF = 4, + BITMAP = 5, + GORILLA_V1 = 6, + REGULAR = 7, GORILLA = 8, ZIGZAG = 9, + FREQ = 10, SPRINTZ = 12, + INVALID_ENCODING = 255, }; // 压缩类型。SNAPPY/GZIP/LZO/LZ4 取决于构建选项;默认压缩为 LZ4。 @@ -54,7 +64,11 @@ enum CompressionType : uint8_t { SNAPPY = 1, GZIP = 2, LZO = 3, + SDT = 4, + PAA = 5, + PLA = 6, LZ4 = 7, + INVALID_COMPRESSION = 255, }; // 列在表 schema 内的角色。 @@ -297,8 +311,13 @@ uint8_t common::get_global_compression(); // 时间列的编码/压缩(数据类型固定为 INT64)。 int common::set_global_time_encoding(uint8_t encoding); int common::set_global_time_compression(uint8_t compression); +uint8_t common::get_global_time_encoding(); +uint8_t common::get_global_time_compression(); ``` +全局压缩支持 `UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO` 和 `LZ4`。 +压缩枚举中也包含 `SDT`、`PAA`、`PLA` 等历史值,但全局压缩 setter 会拒绝这些值。 + ## 读取接口 ### Tsfile Reader ```cpp @@ -335,6 +354,16 @@ class TsFileReader { * @return 成功时返回 0,失败时返回 errno_define.h 中的非零错误码。 */ int query(storage::QueryExpression *qe, ResultSet *&ret_qds); + /** + * @brief 通过路径列表、起始时间和结束时间查询 tsfile。用于 tree 模型。 + * + * @param [in] path_list 完整路径列表。 + * @param [in] start_time 起始时间。 + * @param [in] end_time 结束时间。 + * @param [out] result_set 查询结果集。 + */ + int query(std::vector& path_list, int64_t start_time, + int64_t end_time, ResultSet*& result_set); /** * @brief 通过表名、列名、起始时间和结束时间查询 tsfile。 * @@ -346,7 +375,7 @@ class TsFileReader { */ int query(const std::string &table_name, const std::vector &columns_names, int64_t start_time, - int64_t end_time, ResultSet *&result_set); + int64_t end_time, ResultSet *&result_set, int batch_size = -1); /** * @brief 通过表名、列名、开始时间、结束时间和标签过滤器查询 tsfile。 @@ -360,7 +389,14 @@ class TsFileReader { */ int query(const std::string& table_name, const std::vector& columns_names, int64_t start_time, - int64_t end_time, ResultSet*& result_set, Filter* tag_filter); + int64_t end_time, ResultSet*& result_set, Filter* tag_filter, + int batch_size = 0); + + /** + * @brief 按行查询 tree 模型序列,支持 offset/limit。 + */ + int queryByRow(std::vector& path_list, int offset, int limit, + ResultSet*& result_set); /** * @brief 按行查询表,支持偏移量/行数限制下推与可选的标签过滤。 @@ -379,12 +415,39 @@ class TsFileReader { int limit, ResultSet*& result_set, Filter* tag_filter = nullptr, int batch_size = 0); + /** + * @brief 按测点名在时间范围内查询 tree 模型数据。 + */ + int query_table_on_tree(const std::vector& measurement_names, + int64_t start_time, int64_t end_time, + ResultSet*& result_set); + /** * @brief 销毁结果集,该方法应在查询完成并使用完 result_set 后调用。 * * @param qds 查询结果集。 */ void destroy_query_data_set(ResultSet *qds); + + ResultSet* read_timeseries( + const std::shared_ptr& device_id, + const std::vector& measurement_name); + + std::vector> get_all_devices( + std::string table_name); + + std::vector> get_all_device_ids(); + + std::vector> get_all_devices(); + + int get_timeseries_schema(std::shared_ptr device_id, + std::vector& result); + + DeviceTimeseriesMetadataMap get_timeseries_metadata( + const std::vector>& device_ids); + + DeviceTimeseriesMetadataMap get_timeseries_metadata(); + /** * @brief 根据表名获取表的模式信息。 * @@ -401,6 +464,11 @@ class TsFileReader { std::vector> get_all_table_schemas(); }; ``` + +`DeviceTimeseriesMetadataMap` 是 `get_timeseries_metadata()` 返回的元数据 +map:`std::map, +std::vector>, IDeviceIDComparator>`。 + ### ResultSet ```cpp /** @@ -540,6 +608,8 @@ class TagFilterBuilder { Filter* reg_exp(const std::string& columnName, const std::string& value); Filter* not_reg_exp(const std::string& columnName, const std::string& value); + Filter* is_null(const std::string& columnName); + Filter* is_not_null(const std::string& columnName); Filter* between_and(const std::string& columnName, const std::string& lower, const std::string& upper); Filter* not_between_and(const std::string& columnName, @@ -550,4 +620,4 @@ class TagFilterBuilder { static Filter* or_filter(Filter* left, Filter* right); static Filter* not_filter(Filter* filter); }; -``` \ No newline at end of file +``` diff --git a/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md b/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md index c94b690c7..6eb0d9f3c 100644 --- a/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md +++ b/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md @@ -45,10 +45,16 @@ class TSEncoding(IntEnum): PLAIN = 0 # 所有类型 DICTIONARY = 1 # STRING、TEXT RLE = 2 # INT32、INT64、TIMESTAMP、DATE + DIFF = 3 TS_2DIFF = 4 # INT32、INT64、TIMESTAMP、DATE、FLOAT、DOUBLE + BITMAP = 5 + GORILLA_V1 = 6 + REGULAR = 7 GORILLA = 8 # INT32、INT64、TIMESTAMP、DATE、FLOAT、DOUBLE ZIGZAG = 9 # INT32、INT64 + CHIMP = 11 # INT32、INT64、TIMESTAMP、DATE、FLOAT、DOUBLE SPRINTZ = 12 # INT32、INT64、FLOAT、DOUBLE + RLBE = 13 # INT32、INT64、TIMESTAMP、DATE、FLOAT、DOUBLE class Compressor(IntEnum): """ @@ -58,7 +64,12 @@ class Compressor(IntEnum): SNAPPY = 1 GZIP = 2 LZO = 3 + SDT = 4 + PAA = 5 + PLA = 6 LZ4 = 7 + ZSTD = 8 + LZMA2 = 9 class ColumnCategory(IntEnum): """ @@ -99,6 +110,32 @@ class ResultSetMetaData: def __init__(self, column_list: List[str], data_types: List[TSDataType]) +@dataclass(frozen=True) +class DeviceID: + path: Optional[str] + table_name: Optional[str] + segments: tuple[Optional[str], ...] + +@dataclass(frozen=True) +class TimeseriesStatistic: + has_statistic: bool + row_count: int + start_time: int + end_time: int + +@dataclass(frozen=True) +class TimeseriesMetadata: + measurement_name: str + data_type: TSDataType + chunk_meta_count: int + statistic: TimeseriesStatistic + timeline_statistic: TimeseriesStatistic + +@dataclass(frozen=True) +class DeviceTimeseriesMetadataGroup: + table_name: Optional[str] + segments: tuple[Optional[str], ...] + timeseries: list[TimeseriesMetadata] ``` @@ -106,7 +143,7 @@ class ResultSetMetaData: ## 写入接口 -### TsFileWriter +### TsFileTableWriter ```python class TsFileTableWriter: @@ -137,6 +174,14 @@ class TsFileTableWriter: """ def write_dataframe(self, dataframe: pandas.DataFrame) + """ + 将 pyarrow RecordBatch 或 Table 写入表中。数据必须包含时间列,并且列需要与表 + schema 匹配。 + :param data: pyarrow.RecordBatch 或 pyarrow.Table。 + :return: 无返回值。 + """ + def write_arrow_batch(self, data) + """ 将缓冲数据刷新到磁盘。 :return: 无返回值。 @@ -157,7 +202,6 @@ class TsFileTableWriter: ``` - ### Tablet definition @@ -227,7 +271,9 @@ set_tsfile_config({ | `FLOAT`、`DOUBLE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ` | `GORILLA` | | `STRING`、`TEXT` | `PLAIN`、`DICTIONARY` | `PLAIN` | -压缩适用于所有数据类型:`UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`(默认 `LZ4`)。 +时间列使用全局时间配置,支持 `PLAIN`、`TS_2DIFF`、`GORILLA`、`ZIGZAG`、`RLE` 或 `SPRINTZ`。 + +压缩适用于所有数据类型:`UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`(默认 `LZ4`)。Python 枚举还暴露 `CHIMP`、`RLBE`、`ZSTD`、`LZMA2` 等值,但 `origin/develop` 当前的 writer/config 转换会拒绝它们。 ## 读取接口 @@ -253,11 +299,39 @@ class TsFileReader: :param column_names: 要检索的列名列表。 :param start_time: 查询范围的起始时间(默认:int64 最小值)。 :param end_time: 查询范围的结束时间(默认:int64 最大值)。 + :param tag_filter: 可选的 table 模型 TAG 列谓词。 + :param batch_size: <= 0 逐行返回;> 0 按该大小返回数据块。 :return: 查询结果集处理器。 """ def query_table(self, table_name : str, column_names : List[str], start_time : int = np.iinfo(np.int64).min, - end_time: int = np.iinfo(np.int64).max) -> ResultSet + end_time: int = np.iinfo(np.int64).max, + tag_filter = None, batch_size : int = 0) -> ResultSet + + """ + 对 tree 模型测点列执行时间范围查询。 + + :param column_names: 要检索的测点名列表。 + :param start_time: 查询范围的起始时间。 + :param end_time: 查询范围的结束时间。 + :return: 查询结果集处理器。 + """ + def query_table_on_tree(self, column_names : List[str], + start_time : int = np.iinfo(np.int64).min, + end_time : int = np.iinfo(np.int64).max) -> ResultSet + + """ + 按行查询 tree 模型数据,支持 offset/limit。 + + :param device_ids: 要查询的设备标识列表。 + :param measurement_names: 要检索的测点名列表。 + :param offset: 需要跳过的起始行数。 + :param limit: 最多返回行数;< 0 表示不限制。 + :return: 查询结果集处理器。 + """ + def query_tree_by_row(self, device_ids : List[str], + measurement_names : List[str], + offset : int = 0, limit : int = -1) -> ResultSet """ 按行查询表,支持偏移量/行数限制下推与可选的标签过滤。标签谓词把查询限定到 @@ -277,6 +351,18 @@ class TsFileReader: offset : int = 0, limit : int = -1, tag_filter = None, batch_size : int = 0) -> ResultSet + """ + 对单个设备执行 tree 模型时间范围查询。 + + :param device_name: 设备标识。 + :param sensor_list: 要检索的测点名列表。 + :param start_time: 查询起始时间。 + :param end_time: 查询结束时间。 + :return: 查询结果集处理器。 + """ + def query_timeseries(self, device_name : str, sensor_list : List[str], + start_time : int = 0, end_time : int = 0) -> ResultSet + """ 获取指定表的模式信息。 @@ -292,6 +378,31 @@ class TsFileReader: """ def get_all_table_schemas(self) -> dict[str, TableSchema] + """ + 获取所有 tree 模型 timeseries schema,按设备分组。 + + :return: DeviceSchema 对象列表。 + """ + def get_all_timeseries_schemas(self) -> list[DeviceSchema] + + """ + 获取文件中的所有设备标识。 + + :return: DeviceID(path, table_name, segments) 列表。 + """ + def get_all_devices(self) -> List[DeviceID] + + """ + 获取所有设备或指定设备的 per-timeseries metadata。 + + :param device_ids: None 表示全部设备,[] 表示空结果,或传入 DeviceID / + 路径兼容的设备标识列表。 + :return: dict,key 为设备 segment tuple,value 为 DeviceTimeseriesMetadataGroup。 + """ + def get_timeseries_metadata( + self, device_ids: Optional[List] = None + ) -> Dict[tuple, DeviceTimeseriesMetadataGroup] + """ 关闭 TsFile 读取器。如果读取器中有活动的结果集,它们将失效。 """ @@ -332,6 +443,12 @@ class ResultSet: """ def read_data_frame(self, max_row_num : int = 1024) -> DataFrame + """ + 将下一个批结果读取为 pyarrow.Table。没有更多 TsBlock 批时返回 None。 + 仅适用于通过 batch_size > 0 创建的结果集。 + """ + def read_arrow_batch(self) -> Optional[pyarrow.Table] + """ 从查询结果集中按索引获取值。 @@ -376,6 +493,9 @@ class ResultSet: """ def close(self) + def __enter__(self) + def __exit__(self, exc_type, exc_val, exc_tb) + ``` diff --git a/src/zh/UserGuide/latest/DataFrame/TsFileDataFrame.md b/src/zh/UserGuide/latest/DataFrame/TsFileDataFrame.md index 9fec27c88..263071bc6 100644 --- a/src/zh/UserGuide/latest/DataFrame/TsFileDataFrame.md +++ b/src/zh/UserGuide/latest/DataFrame/TsFileDataFrame.md @@ -59,9 +59,11 @@ data.values # -> np.ndarray, shape (N, 2):N | 示例 | 操作 | 返回类型 | |---|---|---| -| `TsFileDataFrame(paths)` | 加载文件 / 文件列表 / 目录 | `TsFileDataFrame` | +| `TsFileDataFrame(paths, show_progress=True)` | 加载文件 / 文件列表 / 目录。设置 `show_progress=False` 可关闭 stderr 上的加载进度 | `TsFileDataFrame` | | `len(df)` | 时间序列总数 | `int` | +| `df.model` | 已加载文件模型:`"table"` 或 `"tree"` | `str` | | `df.list_timeseries("weather")` | 获取序列名,可按前缀筛选 | `List[str]` | +| `df.list_timeseries_metadata("weather")` | 获取序列元数据,可按前缀筛选 | `pandas.DataFrame` | | `df["weather.Beijing.humidity"]`、`df[0]`、`df[-1]` | 获取单条序列 | `Timeseries` | | `df["city"]` | 获取某元数据列(标签 / `field` / `start_time` / `end_time` / `count`) | `pandas.Series` | | `df[0:3]`、`df[[0, 2, 5]]` | 按整数位置取子集视图:连续区间(`0:3`)或所列位置(`[0, 2, 5]`);位置即打印的 `index` 列 | `TsFileDataFrame` | @@ -132,6 +134,7 @@ from tsfile import TsFileDataFrame df = TsFileDataFrame(["data/weather.tsfile", "data/sensor.tsfile"]) df = TsFileDataFrame("data/") # 递归查找目录下所有 .tsfile +df = TsFileDataFrame("data/", show_progress=False) print(df) ``` @@ -172,7 +175,21 @@ TsFileDataFrame(table model, 972 time series, 5 files) ['weather.Beijing.humidity', 'weather.Beijing.temperature'] ``` -若需查看起止时间、点数等元信息,可打印 DataFrame(或其子集)——见[DataFrame 的展示](#dataframe-的展示)。 +若需查看起止时间、点数等元信息,可打印 DataFrame(或其子集)——见[DataFrame 的展示](#dataframe-的展示)——也可以调用 +`list_timeseries_metadata()`。 + +## 查看元数据 + +`list_timeseries_metadata(path_prefix="")` 返回以序列名为索引的 pandas DataFrame。 +其中包含 `field`、`start_time`、`end_time`、`count` 以及设备标签列。对于 +table 模型文件,还会包含 `table` 列。 + +```python +meta = df.list_timeseries_metadata() +weather_meta = df.list_timeseries_metadata("weather") + +meta[["field", "start_time", "end_time", "count"]].head() +``` ## 选取序列 diff --git a/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md b/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md index de6d39796..9d3b5ea71 100644 --- a/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md +++ b/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md @@ -32,10 +32,12 @@ typedef enum { TS_DATATYPE_FLOAT = 3, TS_DATATYPE_DOUBLE = 4, TS_DATATYPE_TEXT = 5, + TS_DATATYPE_VECTOR = 6, TS_DATATYPE_TIMESTAMP = 8, TS_DATATYPE_DATE = 9, TS_DATATYPE_BLOB = 10, TS_DATATYPE_STRING = 11, + TS_DATATYPE_NULL_TYPE = 254, TS_DATATYPE_INVALID = 255 } TSDataType; @@ -89,9 +91,45 @@ typedef struct result_set_meta_data { TSDataType* data_types; int column_num; } ResultSetMetaData; + +typedef struct arrow_schema ArrowSchema; +typedef struct arrow_array ArrowArray; + +typedef struct TsFileStatisticBase { + bool has_statistic; + TSDataType type; + int32_t row_count; + int64_t start_time; + int64_t end_time; +} TsFileStatisticBase; + +typedef struct TsFileBoolStatistic { TsFileStatisticBase base; double sum; bool first_bool; bool last_bool; } TsFileBoolStatistic; +typedef struct TsFileIntStatistic { TsFileStatisticBase base; double sum; int64_t min_int64; int64_t max_int64; int64_t first_int64; int64_t last_int64; } TsFileIntStatistic; +typedef struct TsFileFloatStatistic { TsFileStatisticBase base; double sum; double min_float64; double max_float64; double first_float64; double last_float64; } TsFileFloatStatistic; +typedef struct TsFileStringStatistic { TsFileStatisticBase base; char* str_min; char* str_max; char* str_first; char* str_last; } TsFileStringStatistic; +typedef struct TsFileTextStatistic { TsFileStatisticBase base; char* str_first; char* str_last; } TsFileTextStatistic; + +typedef union TimeseriesStatisticUnion { + TsFileBoolStatistic bool_s; + TsFileIntStatistic int_s; + TsFileFloatStatistic float_s; + TsFileStringStatistic string_s; + TsFileTextStatistic text_s; +} TimeseriesStatisticUnion; + +typedef struct TimeseriesStatistic { + TimeseriesStatisticUnion u; +} TimeseriesStatistic; + +#define tsfile_statistic_base(s) ((TsFileStatisticBase*)&(s)->u) ``` > `ColumnSchema` 不携带编码/压缩:写入时列遵循全局默认值(见[配置](#配置编码与压缩)),读取时按文件中的实际配置解码。 +> 上面列出的编码和压缩常量是当前 writer/configuration 路径实际接受的值。 +> +> `TimeseriesStatistic` 是 `tsfile_cwrapper.h` 中的带标签 union。可通过 +> `tsfile_statistic_base(&statistic)` 读取通用字段,再根据数据类型读取 +> `int_s`、`float_s`、`bool_s`、`string_s` 或 `text_s`。 ## 写入接口 @@ -207,9 +245,11 @@ ERRNO tablet_add_timestamp(Tablet tablet, uint32_t row_index, * * @param value [输入] 以 '\0' 结尾的字符串,调用者保留所有权。 */ -ERRNO tablet_add_value_by_name_string(Tablet tablet, uint32_t row_index, - const char* column_name, - const char* value); +ERRNO tablet_add_value_by_name_string_with_len(Tablet tablet, + uint32_t row_index, + const char* column_name, + const char* value, + int value_len); // 支持多种数据类型插入 ERRNO tablet_add_value_by_name_int32_t(Tablet tablet, uint32_t row_index, @@ -236,9 +276,11 @@ ERRNO tablet_add_value_by_name_bool(Tablet tablet, uint32_t row_index, * * @param value [输入] 字符串会被内部复制。 */ -ERRNO tablet_add_value_by_index_string(Tablet tablet, uint32_t row_index, - uint32_t column_index, - const char* value); +ERRNO tablet_add_value_by_index_string_with_len(Tablet tablet, + uint32_t row_index, + uint32_t column_index, + const char* value, + int value_len); ERRNO tablet_add_value_by_index_int32_t(Tablet tablet, uint32_t row_index, @@ -314,6 +356,8 @@ uint8_t get_global_time_compression(); | `FLOAT`、`DOUBLE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ` | `GORILLA` | | `STRING`、`TEXT` | `PLAIN`、`DICTIONARY` | `PLAIN` | +时间列使用全局时间配置,支持 `PLAIN`、`TS_2DIFF`、`GORILLA`、`ZIGZAG`、`RLE` 或 `SPRINTZ`。 + 压缩适用于所有数据类型:`UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`(默认 `LZ4`)。 ```C @@ -387,6 +431,31 @@ bool tsfile_result_set_next(ResultSet result_set, ERRNO* error_code); void free_tsfile_result_set(ResultSet* result_set); ``` +### Tree 查询 + +```C +/** + * @brief 按测点名在时间范围内查询 tree 模型数据。 + */ +ResultSet tsfile_query_table_on_tree(TsFileReader reader, char** columns, + uint32_t column_num, Timestamp start_time, + Timestamp end_time, ERRNO* err_code); + +/** + * @brief 按行查询 tree 模型数据,支持 offset/limit。 + * + * @param device_ids 设备标识数组。 + * @param measurement_names 测点名数组。 + * @param offset 需要跳过的起始行数(>= 0)。 + * @param limit 最多返回行数;< 0 表示不限制。 + */ +ResultSet tsfile_reader_query_tree_by_row(TsFileReader reader, + char** device_ids, int device_ids_len, + char** measurement_names, + int measurement_names_len, int offset, + int limit, ERRNO* err_code); +``` + ### 按标签过滤 @@ -409,6 +478,8 @@ typedef enum { TAG_FILTER_GTEQ = 5, // 列 >= 值 TAG_FILTER_REGEXP = 6, // 列匹配正则 值 TAG_FILTER_NOT_REGEXP = 7, // 列不匹配正则 值 + TAG_FILTER_IS_NULL = 8, // 列为空 + TAG_FILTER_IS_NOT_NULL = 9, // 列不为空 } TagFilterOp; /** @@ -417,7 +488,7 @@ typedef enum { * @param reader [输入] 有效的 TsFileReader 句柄。 * @param table_name [输入] 其 schema 定义了这些标签列的表名。 * @param column_name [输入] 要过滤的标签列名。 - * @param value [输入] 比较值(标签列为 STRING 类型)。 + * @param value [输入] 比较值(IS NULL / IS NOT NULL 会忽略该参数)。 * @param op [输入] 比较运算符(TagFilterOp)。 * @param err_code [输出] 成功返回 RET_OK(0),否则返回 errno_define_c.h 中的错误码。 * @return 成功返回 TagFilterHandle,失败返回 NULL。 @@ -437,6 +508,20 @@ TagFilterHandle tsfile_tag_filter_between(TsFileReader reader, const char* lower, const char* upper, bool is_not, ERRNO* err_code); +// 常用单列谓词的便捷构造函数。 +TagFilterHandle tsfile_tag_filter_eq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_neq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_lt(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_lteq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_gt(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_gteq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); + // 组合谓词。AND/OR/NOT 会接管其子节点的所有权,只需释放根节点。 TagFilterHandle tsfile_tag_filter_and(TagFilterHandle left, TagFilterHandle right); TagFilterHandle tsfile_tag_filter_or(TagFilterHandle left, TagFilterHandle right); @@ -493,6 +578,18 @@ ResultSet tsfile_query_table_with_tag_filter( TagFilterHandle tag_filter, int batch_size, ERRNO* err_code); ``` +### 将批结果读取为 Arrow + +对 `batch_size > 0` 创建的批查询结果集,可以使用 Arrow C Data Interface 读取 +`ArrowArray` 和 `ArrowSchema`。调用者拥有返回的 Arrow 对象,使用完后必须调用 +它们的 `release` 回调。 + +```C +ERRNO tsfile_result_set_get_next_tsblock_as_arrow(ResultSet result_set, + ArrowArray* out_array, + ArrowSchema* out_schema); +``` + 示例——只读取 `region` 标签等于 `shanghai` 的设备的 `temperature`: ```C @@ -648,4 +745,3 @@ TableSchema* tsfile_reader_get_all_table_schemas(TsFileReader reader, void free_table_schema(TableSchema schema); ``` - diff --git a/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md b/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md index 50de78b47..5e18d2dac 100644 --- a/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md +++ b/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md @@ -31,10 +31,14 @@ enum TSDataType : uint8_t { FLOAT = 3, DOUBLE = 4, TEXT = 5, + VECTOR = 6, + UNKNOWN = 7, TIMESTAMP = 8, DATE = 9, BLOB = 10, STRING = 11, + NULL_TYPE = 254, + INVALID_DATATYPE = 255, }; // 值编码。各编码适用于哪些类型见下表。 @@ -42,10 +46,16 @@ enum TSEncoding : uint8_t { PLAIN = 0, DICTIONARY = 1, RLE = 2, + DIFF = 3, TS_2DIFF = 4, + BITMAP = 5, + GORILLA_V1 = 6, + REGULAR = 7, GORILLA = 8, ZIGZAG = 9, + FREQ = 10, SPRINTZ = 12, + INVALID_ENCODING = 255, }; // 压缩类型。SNAPPY/GZIP/LZO/LZ4 取决于构建选项;默认压缩为 LZ4。 @@ -54,7 +64,11 @@ enum CompressionType : uint8_t { SNAPPY = 1, GZIP = 2, LZO = 3, + SDT = 4, + PAA = 5, + PLA = 6, LZ4 = 7, + INVALID_COMPRESSION = 255, }; // 列在表 schema 内的角色。 @@ -297,8 +311,13 @@ uint8_t common::get_global_compression(); // 时间列的编码/压缩(数据类型固定为 INT64)。 int common::set_global_time_encoding(uint8_t encoding); int common::set_global_time_compression(uint8_t compression); +uint8_t common::get_global_time_encoding(); +uint8_t common::get_global_time_compression(); ``` +全局压缩支持 `UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO` 和 `LZ4`。 +压缩枚举中也包含 `SDT`、`PAA`、`PLA` 等历史值,但全局压缩 setter 会拒绝这些值。 + ## 读取接口 ### Tsfile Reader ```cpp @@ -335,6 +354,16 @@ class TsFileReader { * @return 成功时返回 0,失败时返回 errno_define.h 中的非零错误码。 */ int query(storage::QueryExpression *qe, ResultSet *&ret_qds); + /** + * @brief 通过路径列表、起始时间和结束时间查询 tsfile。用于 tree 模型。 + * + * @param [in] path_list 完整路径列表。 + * @param [in] start_time 起始时间。 + * @param [in] end_time 结束时间。 + * @param [out] result_set 查询结果集。 + */ + int query(std::vector& path_list, int64_t start_time, + int64_t end_time, ResultSet*& result_set); /** * @brief 通过表名、列名、起始时间和结束时间查询 tsfile。 * @@ -346,7 +375,7 @@ class TsFileReader { */ int query(const std::string &table_name, const std::vector &columns_names, int64_t start_time, - int64_t end_time, ResultSet *&result_set); + int64_t end_time, ResultSet *&result_set, int batch_size = -1); /** * @brief 通过表名、列名、开始时间、结束时间和标签过滤器查询 tsfile。 @@ -360,7 +389,14 @@ class TsFileReader { */ int query(const std::string& table_name, const std::vector& columns_names, int64_t start_time, - int64_t end_time, ResultSet*& result_set, Filter* tag_filter); + int64_t end_time, ResultSet*& result_set, Filter* tag_filter, + int batch_size = 0); + + /** + * @brief 按行查询 tree 模型序列,支持 offset/limit。 + */ + int queryByRow(std::vector& path_list, int offset, int limit, + ResultSet*& result_set); /** * @brief 按行查询表,支持偏移量/行数限制下推与可选的标签过滤。 @@ -379,12 +415,39 @@ class TsFileReader { int limit, ResultSet*& result_set, Filter* tag_filter = nullptr, int batch_size = 0); + /** + * @brief 按测点名在时间范围内查询 tree 模型数据。 + */ + int query_table_on_tree(const std::vector& measurement_names, + int64_t start_time, int64_t end_time, + ResultSet*& result_set); + /** * @brief 销毁结果集,该方法应在查询完成并使用完 result_set 后调用。 * * @param qds 查询结果集。 */ void destroy_query_data_set(ResultSet *qds); + + ResultSet* read_timeseries( + const std::shared_ptr& device_id, + const std::vector& measurement_name); + + std::vector> get_all_devices( + std::string table_name); + + std::vector> get_all_device_ids(); + + std::vector> get_all_devices(); + + int get_timeseries_schema(std::shared_ptr device_id, + std::vector& result); + + DeviceTimeseriesMetadataMap get_timeseries_metadata( + const std::vector>& device_ids); + + DeviceTimeseriesMetadataMap get_timeseries_metadata(); + /** * @brief 根据表名获取表的模式信息。 * @@ -401,6 +464,11 @@ class TsFileReader { std::vector> get_all_table_schemas(); }; ``` + +`DeviceTimeseriesMetadataMap` 是 `get_timeseries_metadata()` 返回的元数据 +map:`std::map, +std::vector>, IDeviceIDComparator>`。 + ### ResultSet ```cpp /** @@ -540,6 +608,8 @@ class TagFilterBuilder { Filter* reg_exp(const std::string& columnName, const std::string& value); Filter* not_reg_exp(const std::string& columnName, const std::string& value); + Filter* is_null(const std::string& columnName); + Filter* is_not_null(const std::string& columnName); Filter* between_and(const std::string& columnName, const std::string& lower, const std::string& upper); Filter* not_between_and(const std::string& columnName, @@ -550,4 +620,4 @@ class TagFilterBuilder { static Filter* or_filter(Filter* left, Filter* right); static Filter* not_filter(Filter* filter); }; -``` \ No newline at end of file +``` diff --git a/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md b/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md index c94b690c7..6eb0d9f3c 100644 --- a/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md +++ b/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md @@ -45,10 +45,16 @@ class TSEncoding(IntEnum): PLAIN = 0 # 所有类型 DICTIONARY = 1 # STRING、TEXT RLE = 2 # INT32、INT64、TIMESTAMP、DATE + DIFF = 3 TS_2DIFF = 4 # INT32、INT64、TIMESTAMP、DATE、FLOAT、DOUBLE + BITMAP = 5 + GORILLA_V1 = 6 + REGULAR = 7 GORILLA = 8 # INT32、INT64、TIMESTAMP、DATE、FLOAT、DOUBLE ZIGZAG = 9 # INT32、INT64 + CHIMP = 11 # INT32、INT64、TIMESTAMP、DATE、FLOAT、DOUBLE SPRINTZ = 12 # INT32、INT64、FLOAT、DOUBLE + RLBE = 13 # INT32、INT64、TIMESTAMP、DATE、FLOAT、DOUBLE class Compressor(IntEnum): """ @@ -58,7 +64,12 @@ class Compressor(IntEnum): SNAPPY = 1 GZIP = 2 LZO = 3 + SDT = 4 + PAA = 5 + PLA = 6 LZ4 = 7 + ZSTD = 8 + LZMA2 = 9 class ColumnCategory(IntEnum): """ @@ -99,6 +110,32 @@ class ResultSetMetaData: def __init__(self, column_list: List[str], data_types: List[TSDataType]) +@dataclass(frozen=True) +class DeviceID: + path: Optional[str] + table_name: Optional[str] + segments: tuple[Optional[str], ...] + +@dataclass(frozen=True) +class TimeseriesStatistic: + has_statistic: bool + row_count: int + start_time: int + end_time: int + +@dataclass(frozen=True) +class TimeseriesMetadata: + measurement_name: str + data_type: TSDataType + chunk_meta_count: int + statistic: TimeseriesStatistic + timeline_statistic: TimeseriesStatistic + +@dataclass(frozen=True) +class DeviceTimeseriesMetadataGroup: + table_name: Optional[str] + segments: tuple[Optional[str], ...] + timeseries: list[TimeseriesMetadata] ``` @@ -106,7 +143,7 @@ class ResultSetMetaData: ## 写入接口 -### TsFileWriter +### TsFileTableWriter ```python class TsFileTableWriter: @@ -137,6 +174,14 @@ class TsFileTableWriter: """ def write_dataframe(self, dataframe: pandas.DataFrame) + """ + 将 pyarrow RecordBatch 或 Table 写入表中。数据必须包含时间列,并且列需要与表 + schema 匹配。 + :param data: pyarrow.RecordBatch 或 pyarrow.Table。 + :return: 无返回值。 + """ + def write_arrow_batch(self, data) + """ 将缓冲数据刷新到磁盘。 :return: 无返回值。 @@ -157,7 +202,6 @@ class TsFileTableWriter: ``` - ### Tablet definition @@ -227,7 +271,9 @@ set_tsfile_config({ | `FLOAT`、`DOUBLE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ` | `GORILLA` | | `STRING`、`TEXT` | `PLAIN`、`DICTIONARY` | `PLAIN` | -压缩适用于所有数据类型:`UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`(默认 `LZ4`)。 +时间列使用全局时间配置,支持 `PLAIN`、`TS_2DIFF`、`GORILLA`、`ZIGZAG`、`RLE` 或 `SPRINTZ`。 + +压缩适用于所有数据类型:`UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`(默认 `LZ4`)。Python 枚举还暴露 `CHIMP`、`RLBE`、`ZSTD`、`LZMA2` 等值,但 `origin/develop` 当前的 writer/config 转换会拒绝它们。 ## 读取接口 @@ -253,11 +299,39 @@ class TsFileReader: :param column_names: 要检索的列名列表。 :param start_time: 查询范围的起始时间(默认:int64 最小值)。 :param end_time: 查询范围的结束时间(默认:int64 最大值)。 + :param tag_filter: 可选的 table 模型 TAG 列谓词。 + :param batch_size: <= 0 逐行返回;> 0 按该大小返回数据块。 :return: 查询结果集处理器。 """ def query_table(self, table_name : str, column_names : List[str], start_time : int = np.iinfo(np.int64).min, - end_time: int = np.iinfo(np.int64).max) -> ResultSet + end_time: int = np.iinfo(np.int64).max, + tag_filter = None, batch_size : int = 0) -> ResultSet + + """ + 对 tree 模型测点列执行时间范围查询。 + + :param column_names: 要检索的测点名列表。 + :param start_time: 查询范围的起始时间。 + :param end_time: 查询范围的结束时间。 + :return: 查询结果集处理器。 + """ + def query_table_on_tree(self, column_names : List[str], + start_time : int = np.iinfo(np.int64).min, + end_time : int = np.iinfo(np.int64).max) -> ResultSet + + """ + 按行查询 tree 模型数据,支持 offset/limit。 + + :param device_ids: 要查询的设备标识列表。 + :param measurement_names: 要检索的测点名列表。 + :param offset: 需要跳过的起始行数。 + :param limit: 最多返回行数;< 0 表示不限制。 + :return: 查询结果集处理器。 + """ + def query_tree_by_row(self, device_ids : List[str], + measurement_names : List[str], + offset : int = 0, limit : int = -1) -> ResultSet """ 按行查询表,支持偏移量/行数限制下推与可选的标签过滤。标签谓词把查询限定到 @@ -277,6 +351,18 @@ class TsFileReader: offset : int = 0, limit : int = -1, tag_filter = None, batch_size : int = 0) -> ResultSet + """ + 对单个设备执行 tree 模型时间范围查询。 + + :param device_name: 设备标识。 + :param sensor_list: 要检索的测点名列表。 + :param start_time: 查询起始时间。 + :param end_time: 查询结束时间。 + :return: 查询结果集处理器。 + """ + def query_timeseries(self, device_name : str, sensor_list : List[str], + start_time : int = 0, end_time : int = 0) -> ResultSet + """ 获取指定表的模式信息。 @@ -292,6 +378,31 @@ class TsFileReader: """ def get_all_table_schemas(self) -> dict[str, TableSchema] + """ + 获取所有 tree 模型 timeseries schema,按设备分组。 + + :return: DeviceSchema 对象列表。 + """ + def get_all_timeseries_schemas(self) -> list[DeviceSchema] + + """ + 获取文件中的所有设备标识。 + + :return: DeviceID(path, table_name, segments) 列表。 + """ + def get_all_devices(self) -> List[DeviceID] + + """ + 获取所有设备或指定设备的 per-timeseries metadata。 + + :param device_ids: None 表示全部设备,[] 表示空结果,或传入 DeviceID / + 路径兼容的设备标识列表。 + :return: dict,key 为设备 segment tuple,value 为 DeviceTimeseriesMetadataGroup。 + """ + def get_timeseries_metadata( + self, device_ids: Optional[List] = None + ) -> Dict[tuple, DeviceTimeseriesMetadataGroup] + """ 关闭 TsFile 读取器。如果读取器中有活动的结果集,它们将失效。 """ @@ -332,6 +443,12 @@ class ResultSet: """ def read_data_frame(self, max_row_num : int = 1024) -> DataFrame + """ + 将下一个批结果读取为 pyarrow.Table。没有更多 TsBlock 批时返回 None。 + 仅适用于通过 batch_size > 0 创建的结果集。 + """ + def read_arrow_batch(self) -> Optional[pyarrow.Table] + """ 从查询结果集中按索引获取值。 @@ -376,6 +493,9 @@ class ResultSet: """ def close(self) + def __enter__(self) + def __exit__(self, exc_type, exc_val, exc_tb) + ```