From 5dcfa3513782c5967d09ae9b4969a35cb263e765 Mon Sep 17 00:00:00 2001 From: Jakob Blomer Date: Tue, 8 Sep 2026 16:43:49 +0200 Subject: [PATCH 1/3] [ntuple] add RRuleField common base class Add a new RRuleField internal base class containing the code dealing with I/O customization rules. This class is initially used by RClassField and will later be reused by RSoAField. --- tree/ntuple/inc/ROOT/RField.hxx | 74 ++++--- tree/ntuple/src/RFieldMeta.cxx | 356 ++++++++++++++++---------------- 2 files changed, 225 insertions(+), 205 deletions(-) diff --git a/tree/ntuple/inc/ROOT/RField.hxx b/tree/ntuple/inc/ROOT/RField.hxx index 1429a8e3f0b5c..94aae33e6e9b3 100644 --- a/tree/ntuple/inc/ROOT/RField.hxx +++ b/tree/ntuple/inc/ROOT/RField.hxx @@ -131,17 +131,9 @@ public: size_t GetAlignment() const final { return 0; } }; // RInvalidField -/// The field for a class with dictionary -class RClassField : public RFieldBase { -private: - enum ESubfieldRole { - kBaseClass, - kDataMember, - }; - struct RSubfieldInfo { - ESubfieldRole fRole; - std::size_t fOffset; - }; +/// Base class for fields that are subject to I/O customization rules. Use by the class field and the SoA field. +class RRuleField : public RFieldBase { +protected: // Information to read into the staging area a field that is used as an input to an I/O customization rule struct RStagingItem { /// The field used to read the on-disk data. The fields type may be different from the on-disk type as long @@ -152,19 +144,6 @@ private: /// Prefix used in the subfield names generated for base classes static constexpr const char *kPrefixInherited{":"}; - class RClassDeleter : public RDeleter { - private: - TClass *fClass; - - public: - explicit RClassDeleter(TClass *cl); - void operator()(void *objPtr, bool dtorOnly) final; - }; - - TClass *fClass; - /// Additional information kept for each entry in `fSubfields` - std::vector fSubfieldsInfo; - /// The staging area stores inputs to I/O rules according to the offsets given by the streamer info of /// "TypeName@@Version". The area is allocated depending on I/O rules resp. the source members of the I/O rules. std::unique_ptr fStagingArea; @@ -175,15 +154,15 @@ private: TClass *fStagingClass = nullptr; std::unordered_map fStagingItems; ///< Lookup staging items by member name -private: - RClassField(std::string_view fieldName, const RClassField &source); ///< Used by CloneImpl - RClassField(std::string_view fieldName, TClass *classp); - void Attach(std::unique_ptr child, RSubfieldInfo info); + RRuleField(std::string_view name, std::string_view type, ROOT::ENTupleStructure structure); + + /// Derived classes should return the TClass instance representing the current in-memory layout. + virtual TClass *GetInMemoryClass() const = 0; /// Returns the id of member 'name' in the class field given by 'fieldId', or kInvalidDescriptorId if no such /// member exist. Looks recursively in base classes. - ROOT::DescriptorId_t - LookupMember(const ROOT::RNTupleDescriptor &desc, std::string_view memberName, ROOT::DescriptorId_t classFieldId); + ROOT::DescriptorId_t LookupMember(const ROOT::RNTupleDescriptor &desc, std::string_view memberName, + ROOT::DescriptorId_t classFieldId) const; /// Sets fStagingClass according to the given name and version void SetStagingClass(const std::string &className, unsigned int classVersion); /// If there are rules with inputs (source members), create the staging area according to the TClass instance @@ -196,7 +175,38 @@ private: /// to the class field at hand, to which the fieldDesc descriptor, if provided, must correspond. /// Fields may not have an on-disk representation (e.g., when inserted by schema evolution), in which case the passed /// field descriptor is nullptr. - std::vector FindRules(const ROOT::RFieldDescriptor *fieldDesc); + std::vector FindRules(const ROOT::RFieldDescriptor *fieldDesc) const; +}; + +/// The field for a class with dictionary +class RClassField : public RRuleField { +private: + enum ESubfieldRole { + kBaseClass, + kDataMember, + }; + struct RSubfieldInfo { + ESubfieldRole fRole; + std::size_t fOffset; + }; + + class RClassDeleter : public RDeleter { + private: + TClass *fClass; + + public: + explicit RClassDeleter(TClass *cl); + void operator()(void *objPtr, bool dtorOnly) final; + }; + + TClass *fClass; + /// Additional information kept for each entry in `fSubfields` + std::vector fSubfieldsInfo; + +private: + RClassField(std::string_view fieldName, const RClassField &source); ///< Used by CloneImpl + RClassField(std::string_view fieldName, TClass *classp); + void Attach(std::unique_ptr child, RSubfieldInfo info); protected: std::unique_ptr CloneImpl(std::string_view newName) const final; @@ -211,6 +221,8 @@ protected: std::unique_ptr BeforeConnectPageSource(ROOT::Internal::RPageSource &pageSource) final; void ReconcileOnDiskField(const RNTupleDescriptor &desc) final; + TClass *GetInMemoryClass() const final { return fClass; } + public: RClassField(std::string_view fieldName, std::string_view className); RClassField(RClassField &&other) = default; diff --git a/tree/ntuple/src/RFieldMeta.cxx b/tree/ntuple/src/RFieldMeta.cxx index 4d569a4e6ff1b..a3b4ac96de415 100644 --- a/tree/ntuple/src/RFieldMeta.cxx +++ b/tree/ntuple/src/RFieldMeta.cxx @@ -190,8 +190,188 @@ std::string BuildMapTypeName(ROOT::RMapField::EMapType mapType, const ROOT::RFie } // anonymous namespace +ROOT::RRuleField::RRuleField(std::string_view fieldName, std::string_view typeName, ROOT::ENTupleStructure structure) + : RFieldBase(fieldName, typeName, structure, false /* isSimple */) +{ +} + +ROOT::DescriptorId_t ROOT::RRuleField::LookupMember(const ROOT::RNTupleDescriptor &desc, std::string_view memberName, + ROOT::DescriptorId_t classFieldId) const +{ + auto idSourceMember = desc.FindFieldId(memberName, classFieldId); + if (idSourceMember != ROOT::kInvalidDescriptorId) + return idSourceMember; + + for (const auto &subFieldDesc : desc.GetFieldIterable(classFieldId)) { + const auto &subFieldName = subFieldDesc.GetFieldName(); + if (subFieldName.length() > 2 && subFieldName[0] == ':' && subFieldName[1] == '_') { + idSourceMember = LookupMember(desc, memberName, subFieldDesc.GetId()); + if (idSourceMember != ROOT::kInvalidDescriptorId) + return idSourceMember; + } + } + + return ROOT::kInvalidDescriptorId; +} + +void ROOT::RRuleField::SetStagingClass(const std::string &className, unsigned int classVersion) +{ + TClass::GetClass(className.c_str())->GetStreamerInfo(classVersion); + if (classVersion != GetTypeVersion() || className != GetTypeName()) { + fStagingClass = TClass::GetClass((className + std::string("@@") + std::to_string(classVersion)).c_str()); + if (!fStagingClass) { + // For a rename rule, we may simply ask for the old class name + fStagingClass = TClass::GetClass(className.c_str()); + } + } else { + fStagingClass = GetInMemoryClass(); + } + R__ASSERT(fStagingClass); + R__ASSERT(static_cast(fStagingClass->GetClassVersion()) == classVersion); +} + +void ROOT::RRuleField::PrepareStagingArea(const std::vector &rules, + const ROOT::RNTupleDescriptor &desc, + const ROOT::RFieldDescriptor &classFieldDesc) +{ + std::size_t stagingAreaSize = 0; + for (const auto rule : rules) { + for (auto source : TRangeDynCast(rule->GetSource())) { + auto [itr, isNew] = fStagingItems.emplace(source->GetName(), RStagingItem()); + if (!isNew) { + // This source member has already been processed by another rule (and we only support one type per member) + continue; + } + RStagingItem &stagingItem = itr->second; + + const auto memberFieldId = LookupMember(desc, source->GetName(), classFieldDesc.GetId()); + if (memberFieldId == kInvalidDescriptorId) { + throw RException(R__FAIL(std::string("cannot find on disk rule source member ") + GetTypeName() + "." + + source->GetName())); + } + + auto memberType = source->GetTypeForDeclaration() + source->GetDimensions(); + auto memberField = Create("" /* we don't need a field name */, std::string(memberType)).Unwrap(); + memberField->SetOnDiskId(memberFieldId); + auto fieldZero = std::make_unique(); + Internal::SetAllowFieldSubstitutions(*fieldZero, true); + fieldZero->Attach(std::move(memberField)); + stagingItem.fField = std::move(fieldZero); + + stagingItem.fOffset = fStagingClass->GetDataMemberOffset(source->GetName()); + // Since we successfully looked up the source member in the RNTuple on-disk metadata, we expect it + // to be present in the TClass instance, too. + R__ASSERT(stagingItem.fOffset != TVirtualStreamerInfo::kMissing); + stagingAreaSize = std::max(stagingAreaSize, stagingItem.fOffset + stagingItem.fField->begin()->GetValueSize()); + } + } + + if (stagingAreaSize) { + R__ASSERT(static_cast(stagingAreaSize) <= fStagingClass->Size()); // we may have removed rules + // We use std::make_unique instead of MakeUninitArray to zero-initialize the staging area. + fStagingArea = std::make_unique(stagingAreaSize); + + for (const auto &[_, si] : fStagingItems) { + const auto &memberField = *si.fField->cbegin(); + if (!(memberField.GetTraits() & kTraitTriviallyConstructible)) { + CallConstructValueOn(memberField, fStagingArea.get() + si.fOffset); + } + } + } +} + +std::vector ROOT::RRuleField::FindRules(const ROOT::RFieldDescriptor *fieldDesc) const +{ + ROOT::Detail::TSchemaRuleSet::TMatches rules; + auto cl = GetInMemoryClass(); + + const auto ruleset = cl->GetSchemaRules(); + if (!ruleset) + return rules; + + if (!fieldDesc) { + // If we have no on-disk information for the field, we still process the rules on the current in-memory version + // of the class + rules = ruleset->FindRules(cl->GetName(), cl->GetClassVersion(), cl->GetCheckSum()); + } else { + // We need to change (back) the name normalization from RNTuple to ROOT Meta + std::string normalizedName; + TClassEdit::GetNormalizedName(normalizedName, fieldDesc->GetTypeName()); + // We do have an on-disk field that correspond to the current RClassField instance. Ask for rules matching the + // on-disk version of the field. + if (fieldDesc->GetTypeChecksum()) { + rules = ruleset->FindRules(normalizedName, fieldDesc->GetTypeVersion(), *fieldDesc->GetTypeChecksum()); + } else { + rules = ruleset->FindRules(normalizedName, fieldDesc->GetTypeVersion()); + } + } + + // Cleanup and sort rules + // Check that any any given source member uses the same type in all rules + std::unordered_map sourceNameAndType; + std::size_t nskip = 0; // skip whole-object-rules that were moved to the end of the rules vector + for (auto itr = rules.begin(); itr != rules.end() - nskip;) { + const auto rule = *itr; + + // Erase unknown rule types + if (rule->GetRuleType() != ROOT::TSchemaRule::kReadRule) { + R__LOG_WARNING(ROOT::Internal::NTupleLog()) + << "ignoring I/O customization rule with unsupported type: " << rule->GetRuleType(); + itr = rules.erase(itr); + continue; + } + + bool hasConflictingSourceMembers = false; + for (auto source : TRangeDynCast(rule->GetSource())) { + auto memberType = source->GetTypeForDeclaration() + source->GetDimensions(); + auto [itrSrc, isNew] = sourceNameAndType.emplace(source->GetName(), memberType); + if (!isNew && (itrSrc->second != memberType)) { + R__LOG_WARNING(ROOT::Internal::NTupleLog()) + << "ignoring I/O customization rule due to conflicting source member type: " << itrSrc->second << " vs. " + << memberType << " for member " << source->GetName(); + hasConflictingSourceMembers = true; + break; + } + } + if (hasConflictingSourceMembers) { + itr = rules.erase(itr); + continue; + } + + // Rules targeting the entire object need to be executed at the end + if (rule->GetTarget() == nullptr) { + nskip++; + if (itr != rules.end() - nskip) + std::iter_swap(itr++, rules.end() - nskip); + continue; + } + + ++itr; + } + + return rules; +} + +void ROOT::RRuleField::AddReadCallbacksFromIORule(const TSchemaRule *rule) +{ + auto func = rule->GetReadFunctionPointer(); + if (func == nullptr) { + // Can happen for rename rules + return; + } + fReadCallbacks.emplace_back([func, stagingClass = fStagingClass, stagingArea = fStagingArea.get()](void *target) { + TVirtualObject onfileObj{nullptr}; + onfileObj.fClass = stagingClass; + onfileObj.fObject = stagingArea; + func(static_cast(target), &onfileObj); + onfileObj.fObject = nullptr; // TVirtualObject does not own the value + }); +} + +//------------------------------------------------------------------------------ + ROOT::RClassField::RClassField(std::string_view fieldName, const RClassField &source) - : ROOT::RFieldBase(fieldName, source.GetTypeName(), ROOT::ENTupleStructure::kRecord, false /* isSimple */), + : ROOT::RRuleField(fieldName, source.GetTypeName(), ROOT::ENTupleStructure::kRecord), fClass(source.fClass), fSubfieldsInfo(source.fSubfieldsInfo) { @@ -207,8 +387,7 @@ ROOT::RClassField::RClassField(std::string_view fieldName, std::string_view clas } ROOT::RClassField::RClassField(std::string_view fieldName, TClass *classp) - : ROOT::RFieldBase(fieldName, GetRenormalizedTypeName(classp->GetName()), ROOT::ENTupleStructure::kRecord, - false /* isSimple */), + : ROOT::RRuleField(fieldName, GetRenormalizedTypeName(classp->GetName()), ROOT::ENTupleStructure::kRecord), fClass(classp) { EnsureValidUserClass(fClass, *this, "RClassField"); @@ -290,76 +469,6 @@ void ROOT::RClassField::Attach(std::unique_ptr child, RSubfieldInfo RFieldBase::Attach(std::move(child)); } -std::vector ROOT::RClassField::FindRules(const ROOT::RFieldDescriptor *fieldDesc) -{ - ROOT::Detail::TSchemaRuleSet::TMatches rules; - const auto ruleset = fClass->GetSchemaRules(); - if (!ruleset) - return rules; - - if (!fieldDesc) { - // If we have no on-disk information for the field, we still process the rules on the current in-memory version - // of the class - rules = ruleset->FindRules(fClass->GetName(), fClass->GetClassVersion(), fClass->GetCheckSum()); - } else { - // We need to change (back) the name normalization from RNTuple to ROOT Meta - std::string normalizedName; - TClassEdit::GetNormalizedName(normalizedName, fieldDesc->GetTypeName()); - // We do have an on-disk field that correspond to the current RClassField instance. Ask for rules matching the - // on-disk version of the field. - if (fieldDesc->GetTypeChecksum()) { - rules = ruleset->FindRules(normalizedName, fieldDesc->GetTypeVersion(), *fieldDesc->GetTypeChecksum()); - } else { - rules = ruleset->FindRules(normalizedName, fieldDesc->GetTypeVersion()); - } - } - - // Cleanup and sort rules - // Check that any any given source member uses the same type in all rules - std::unordered_map sourceNameAndType; - std::size_t nskip = 0; // skip whole-object-rules that were moved to the end of the rules vector - for (auto itr = rules.begin(); itr != rules.end() - nskip;) { - const auto rule = *itr; - - // Erase unknown rule types - if (rule->GetRuleType() != ROOT::TSchemaRule::kReadRule) { - R__LOG_WARNING(ROOT::Internal::NTupleLog()) - << "ignoring I/O customization rule with unsupported type: " << rule->GetRuleType(); - itr = rules.erase(itr); - continue; - } - - bool hasConflictingSourceMembers = false; - for (auto source : TRangeDynCast(rule->GetSource())) { - auto memberType = source->GetTypeForDeclaration() + source->GetDimensions(); - auto [itrSrc, isNew] = sourceNameAndType.emplace(source->GetName(), memberType); - if (!isNew && (itrSrc->second != memberType)) { - R__LOG_WARNING(ROOT::Internal::NTupleLog()) - << "ignoring I/O customization rule due to conflicting source member type: " << itrSrc->second << " vs. " - << memberType << " for member " << source->GetName(); - hasConflictingSourceMembers = true; - break; - } - } - if (hasConflictingSourceMembers) { - itr = rules.erase(itr); - continue; - } - - // Rules targeting the entire object need to be executed at the end - if (rule->GetTarget() == nullptr) { - nskip++; - if (itr != rules.end() - nskip) - std::iter_swap(itr++, rules.end() - nskip); - continue; - } - - ++itr; - } - - return rules; -} - std::unique_ptr ROOT::RClassField::CloneImpl(std::string_view newName) const { return std::unique_ptr(new RClassField(newName, *this)); @@ -394,107 +503,6 @@ void ROOT::RClassField::ReadInClusterImpl(RNTupleLocalIndex localIndex, void *to } } -ROOT::DescriptorId_t ROOT::RClassField::LookupMember(const ROOT::RNTupleDescriptor &desc, std::string_view memberName, - ROOT::DescriptorId_t classFieldId) -{ - auto idSourceMember = desc.FindFieldId(memberName, classFieldId); - if (idSourceMember != ROOT::kInvalidDescriptorId) - return idSourceMember; - - for (const auto &subFieldDesc : desc.GetFieldIterable(classFieldId)) { - const auto &subFieldName = subFieldDesc.GetFieldName(); - if (subFieldName.length() > 2 && subFieldName[0] == ':' && subFieldName[1] == '_') { - idSourceMember = LookupMember(desc, memberName, subFieldDesc.GetId()); - if (idSourceMember != ROOT::kInvalidDescriptorId) - return idSourceMember; - } - } - - return ROOT::kInvalidDescriptorId; -} - -void ROOT::RClassField::SetStagingClass(const std::string &className, unsigned int classVersion) -{ - TClass::GetClass(className.c_str())->GetStreamerInfo(classVersion); - if (classVersion != GetTypeVersion() || className != GetTypeName()) { - fStagingClass = TClass::GetClass((className + std::string("@@") + std::to_string(classVersion)).c_str()); - if (!fStagingClass) { - // For a rename rule, we may simply ask for the old class name - fStagingClass = TClass::GetClass(className.c_str()); - } - } else { - fStagingClass = fClass; - } - R__ASSERT(fStagingClass); - R__ASSERT(static_cast(fStagingClass->GetClassVersion()) == classVersion); -} - -void ROOT::RClassField::PrepareStagingArea(const std::vector &rules, - const ROOT::RNTupleDescriptor &desc, - const ROOT::RFieldDescriptor &classFieldDesc) -{ - std::size_t stagingAreaSize = 0; - for (const auto rule : rules) { - for (auto source : TRangeDynCast(rule->GetSource())) { - auto [itr, isNew] = fStagingItems.emplace(source->GetName(), RStagingItem()); - if (!isNew) { - // This source member has already been processed by another rule (and we only support one type per member) - continue; - } - RStagingItem &stagingItem = itr->second; - - const auto memberFieldId = LookupMember(desc, source->GetName(), classFieldDesc.GetId()); - if (memberFieldId == kInvalidDescriptorId) { - throw RException(R__FAIL(std::string("cannot find on disk rule source member ") + GetTypeName() + "." + - source->GetName())); - } - - auto memberType = source->GetTypeForDeclaration() + source->GetDimensions(); - auto memberField = Create("" /* we don't need a field name */, std::string(memberType)).Unwrap(); - memberField->SetOnDiskId(memberFieldId); - auto fieldZero = std::make_unique(); - Internal::SetAllowFieldSubstitutions(*fieldZero, true); - fieldZero->Attach(std::move(memberField)); - stagingItem.fField = std::move(fieldZero); - - stagingItem.fOffset = fStagingClass->GetDataMemberOffset(source->GetName()); - // Since we successfully looked up the source member in the RNTuple on-disk metadata, we expect it - // to be present in the TClass instance, too. - R__ASSERT(stagingItem.fOffset != TVirtualStreamerInfo::kMissing); - stagingAreaSize = std::max(stagingAreaSize, stagingItem.fOffset + stagingItem.fField->begin()->GetValueSize()); - } - } - - if (stagingAreaSize) { - R__ASSERT(static_cast(stagingAreaSize) <= fStagingClass->Size()); // we may have removed rules - // We use std::make_unique instead of MakeUninitArray to zero-initialize the staging area. - fStagingArea = std::make_unique(stagingAreaSize); - - for (const auto &[_, si] : fStagingItems) { - const auto &memberField = *si.fField->cbegin(); - if (!(memberField.GetTraits() & kTraitTriviallyConstructible)) { - CallConstructValueOn(memberField, fStagingArea.get() + si.fOffset); - } - } - } -} - -void ROOT::RClassField::AddReadCallbacksFromIORule(const TSchemaRule *rule) -{ - auto func = rule->GetReadFunctionPointer(); - if (func == nullptr) { - // Can happen for rename rules - return; - } - fReadCallbacks.emplace_back([func, stagingClass = fStagingClass, stagingArea = fStagingArea.get()](void *target) { - TVirtualObject onfileObj{nullptr}; - onfileObj.fClass = stagingClass; - onfileObj.fObject = stagingArea; - func(static_cast(target), &onfileObj); - onfileObj.fObject = nullptr; // TVirtualObject does not own the value - }); -} - std::unique_ptr ROOT::RClassField::BeforeConnectPageSource(ROOT::Internal::RPageSource &pageSource) { std::vector rules; From 1d2f3d2a26277b3e59268d0398e87cd4420259c7 Mon Sep 17 00:00:00 2001 From: Jakob Blomer Date: Tue, 8 Sep 2026 16:49:46 +0200 Subject: [PATCH 2/3] [ntuple] let RSoAField inherit from RRuleField --- tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx | 4 +++- tree/ntuple/src/RFieldMeta.cxx | 5 ++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx b/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx index af503964a865c..b9536af6ea912 100644 --- a/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx +++ b/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx @@ -53,7 +53,7 @@ namespace Experimental { /// /// Since the on-disk representation is a collection of record type, the class version and checksum of the SoA type /// itself is ignored. -class RSoAField : public RFieldBase { +class RSoAField : public RRuleField { class RSoADeleter : public RDeleter { private: TClass *fSoAClass; @@ -116,6 +116,8 @@ protected: void ReconcileOnDiskField(const RNTupleDescriptor &desc) final; + TClass *GetInMemoryClass() const final { return fSoAClass; } + public: RSoAField(std::string_view fieldName, std::string_view className); RSoAField(RSoAField &&other) = default; diff --git a/tree/ntuple/src/RFieldMeta.cxx b/tree/ntuple/src/RFieldMeta.cxx index a3b4ac96de415..f603cacefe8e2 100644 --- a/tree/ntuple/src/RFieldMeta.cxx +++ b/tree/ntuple/src/RFieldMeta.cxx @@ -689,7 +689,7 @@ void ROOT::RClassField::AcceptVisitor(ROOT::Detail::RFieldVisitor &visitor) cons //------------------------------------------------------------------------------ ROOT::Experimental::RSoAField::RSoAField(std::string_view fieldName, const RSoAField &source) - : ROOT::RFieldBase(fieldName, source.GetTypeName(), ROOT::ENTupleStructure::kCollection, false /* isSimple */), + : ROOT::RRuleField(fieldName, source.GetTypeName(), ROOT::ENTupleStructure::kCollection), fSoAClass(source.fSoAClass), fSoAMemberOffsets(source.fSoAMemberOffsets) { @@ -879,8 +879,7 @@ void ROOT::Experimental::RSoAField::CollectRecordMemberFields() } ROOT::Experimental::RSoAField::RSoAField(std::string_view fieldName, TClass *clSoA) - : ROOT::RFieldBase(fieldName, GetRenormalizedTypeName(clSoA->GetName()), ROOT::ENTupleStructure::kCollection, - false /* isSimple */), + : ROOT::RRuleField(fieldName, GetRenormalizedTypeName(clSoA->GetName()), ROOT::ENTupleStructure::kCollection), fSoAClass(clSoA) { static std::once_flag once; From 24bcb18246b24a3f041516750719f82e4b0a7d1a Mon Sep 17 00:00:00 2001 From: Jakob Blomer Date: Wed, 9 Sep 2026 10:20:41 +0200 Subject: [PATCH 3/3] [ntuple] basic support rules in SoA classes Adds support for rename and whole-object rules. Errors out when rules with targets or sources are found. --- tree/ntuple/inc/ROOT/RField.hxx | 4 +- tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx | 13 ++- tree/ntuple/src/RFieldMeta.cxx | 66 ++++++++++++--- tree/ntuple/test/SoAField.hxx | 84 ++++++++++++++++++++ tree/ntuple/test/SoAFieldLinkDef.h | 30 +++++++ tree/ntuple/test/ntuple_soa.cxx | 97 +++++++++++++++++++++-- 6 files changed, 275 insertions(+), 19 deletions(-) diff --git a/tree/ntuple/inc/ROOT/RField.hxx b/tree/ntuple/inc/ROOT/RField.hxx index 94aae33e6e9b3..b1df44972d936 100644 --- a/tree/ntuple/inc/ROOT/RField.hxx +++ b/tree/ntuple/inc/ROOT/RField.hxx @@ -170,7 +170,9 @@ protected: void PrepareStagingArea(const std::vector &rules, const ROOT::RNTupleDescriptor &desc, const ROOT::RFieldDescriptor &classFieldId); /// Register post-read callback corresponding to a ROOT I/O customization rules. - void AddReadCallbacksFromIORule(const TSchemaRule *rule); + /// The sub object offset allows to apply the rule to a nested object within the passed target. + /// This is used to execute rules on base classes and nested classes in a SoA field. + void AddReadCallbacksFromIORule(const TSchemaRule *rule, std::size_t subObjectOffset = 0); /// Given the on-disk information from the page source, find all the I/O customization rules that apply /// to the class field at hand, to which the fieldDesc descriptor, if provided, must correspond. /// Fields may not have an on-disk representation (e.g., when inserted by schema evolution), in which case the passed diff --git a/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx b/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx index b9536af6ea912..cb218662e17ea 100644 --- a/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx +++ b/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx @@ -63,6 +63,13 @@ class RSoAField : public RRuleField { void operator()(void *objPtr, bool dtorOnly) final; }; + // A rule of the SoA type itself or one of its nested SoA types or base classes, togther with the offset + // of the nested object in the SoA type. + struct RRule { + const TSchemaRule *fRule = nullptr; + std::size_t fOffset = 0; + }; + TClass *fSoAClass = nullptr; /// Direct access to the member fields of the underlying record. In case of a nested SoA type, this vector /// contains the contents of the inner fRecordMemberFields, too. Effectively, this record will contain all the @@ -75,6 +82,9 @@ class RSoAField : public RRuleField { std::vector> fRecordMemberDeleters; ROOT::Internal::RColumnIndex fNWritten; + /// Contains the I/O customization rules for fSoAClass and all nested SoA classes and base classes. + std::vector fRules; + /// For reading and writing, the RVecs of the SoA class do not have a dedicated field. The in-memory RVecs of the /// SoA object are used directly with the subfields of the underlying record type. For splitting a SoA class object /// (SplitValue()), however, we need actual RRVecFields so that we can recursively split the in-memory SoA value. @@ -90,7 +100,7 @@ class RSoAField : public RRuleField { RSoAField(std::string_view fieldName, TClass *clSoA); /// Called during construction, picks up the (nested) member fields of the underlying record type(s) and its - /// base classes. + /// base classes. Also fills fRules. void CollectRecordMemberFields(); /// For a nested SoA struct (either as a member of as a base class), use their fRecordMemberFields in this class, /// i.e. "unroll" the vectors in the nested SoA struct into the SoA base class. @@ -114,6 +124,7 @@ protected: void CommitClusterImpl() final { fNWritten = 0; } + std::unique_ptr BeforeConnectPageSource(ROOT::Internal::RPageSource &pageSource) final; void ReconcileOnDiskField(const RNTupleDescriptor &desc) final; TClass *GetInMemoryClass() const final { return fSoAClass; } diff --git a/tree/ntuple/src/RFieldMeta.cxx b/tree/ntuple/src/RFieldMeta.cxx index f603cacefe8e2..fbd13d20ba21f 100644 --- a/tree/ntuple/src/RFieldMeta.cxx +++ b/tree/ntuple/src/RFieldMeta.cxx @@ -352,20 +352,21 @@ std::vector ROOT::RRuleField::FindRules(const ROOT::R return rules; } -void ROOT::RRuleField::AddReadCallbacksFromIORule(const TSchemaRule *rule) +void ROOT::RRuleField::AddReadCallbacksFromIORule(const TSchemaRule *rule, std::size_t subObjectOffset) { auto func = rule->GetReadFunctionPointer(); if (func == nullptr) { // Can happen for rename rules return; } - fReadCallbacks.emplace_back([func, stagingClass = fStagingClass, stagingArea = fStagingArea.get()](void *target) { - TVirtualObject onfileObj{nullptr}; - onfileObj.fClass = stagingClass; - onfileObj.fObject = stagingArea; - func(static_cast(target), &onfileObj); - onfileObj.fObject = nullptr; // TVirtualObject does not own the value - }); + fReadCallbacks.emplace_back( + [func, subObjectOffset, stagingClass = fStagingClass, stagingArea = fStagingArea.get()](void *target) { + TVirtualObject onfileObj{nullptr}; + onfileObj.fClass = stagingClass; + onfileObj.fObject = stagingArea; + func(static_cast(target) + subObjectOffset, &onfileObj); + onfileObj.fObject = nullptr; // TVirtualObject does not own the value + }); } //------------------------------------------------------------------------------ @@ -816,6 +817,11 @@ void ROOT::Experimental::RSoAField::CollectRecordMemberFields() return realRecordMemberFields[recordFieldNameToIdx[name]]; }); + for (const RRule &r : soaBaseField->fRules) { + fRules.emplace_back(r); + fRules.back().fOffset += base->GetDelta(); + } + baseIdx++; } @@ -851,6 +857,11 @@ void ROOT::Experimental::RSoAField::CollectRecordMemberFields() GraftNestedMemberFields(*soaField, dataMember->GetOffset(), [&](const std::string &name) { return realRecordMemberFields[recordFieldNameToIdx[name]]; }); + + for (const RRule &r : soaField->fRules) { + fRules.emplace_back(r); + fRules.back().fOffset += dataMember->GetOffset(); + } } else if (auto vecField = dynamic_cast(dmField.get())) { if (vecField->begin()->GetTypeName() != underlyingField->GetTypeName() || vecField->begin()->GetTypeAlias() != underlyingField->GetTypeAlias()) { @@ -876,6 +887,10 @@ void ROOT::Experimental::RSoAField::CollectRecordMemberFields() if (nDirectRecordSubfields != nMembers) { throw RException(R__FAIL("missing SoA members")); } + + auto schemaRules = FindRules(nullptr); + for (const auto &r : schemaRules) + fRules.emplace_back(RRule{r, 0}); } ROOT::Experimental::RSoAField::RSoAField(std::string_view fieldName, TClass *clSoA) @@ -1021,9 +1036,42 @@ void ROOT::Experimental::RSoAField::ReadGlobalImpl(ROOT::NTupleSize_t globalInde } } +std::unique_ptr +ROOT::Experimental::RSoAField::BeforeConnectPageSource(ROOT::Internal::RPageSource & /*pageSource*/) +{ + // Most of the heavy lifting is done by the actual subfields that map onto the on-disk AoS schema. + // In the SoA field itself, we only need to take care of the in-memory part of the rule processing. + // One complication is that for non-rename rules, we manually need to recurse into base (SoA) classes and + // nested (SoA) classes because we don't have them in the field tree. + + // For now, allow only rename rules and whole-object rules + + const bool hasSources = std::any_of(fRules.begin(), fRules.end(), [](const auto &r) { + return r.fRule->GetSource() && (r.fRule->GetSource()->GetEntries() > 0); + }); + const bool hasTargets = + std::any_of(fRules.begin(), fRules.end(), [](const auto &r) { return r.fRule->GetTarget(); }); + + if (hasSources) { + throw RException(R__FAIL("I/O customization rules with sources are currently unsupported for SoA fields (" + + GetTypeName() + ")")); + } + + if (hasTargets) { + throw RException(R__FAIL("I/O customization rules with targets are currently unsupported for SoA fields (" + + GetTypeName() + ")")); + } + + for (const auto &r : fRules) { + AddReadCallbacksFromIORule(r.fRule, r.fOffset); + } + + return nullptr; +} + void ROOT::Experimental::RSoAField::ReconcileOnDiskField(const RNTupleDescriptor &desc) { - EnsureMatchingOnDiskField(desc, kDiffTypeVersion).ThrowOnError(); + EnsureMatchingOnDiskField(desc, kDiffTypeName | kDiffTypeVersion).ThrowOnError(); } void ROOT::Experimental::RSoAField::ConstructValue(void *where) const diff --git a/tree/ntuple/test/SoAField.hxx b/tree/ntuple/test/SoAField.hxx index e74192aa84c88..d3438f83ff584 100644 --- a/tree/ntuple/test/SoAField.hxx +++ b/tree/ntuple/test/SoAField.hxx @@ -170,4 +170,88 @@ struct SoADerivedFail2 : public SoABase { ClassDefNV(SoADerivedFail2, 2); }; +struct RecordBaseOld { + float fBase; + ClassDefNV(RecordBaseOld, 2); +}; + +struct SoABaseOld { + ROOT::RVec fBase; + ClassDefNV(SoABaseOld, 2); +}; + +struct RecordIntermediateOld : public RecordBaseOld { + float fIntermediate; + ClassDefNV(RecordIntermediateOld, 2); +}; + +struct SoAIntermediateOld : public SoABaseOld { + ROOT::RVec fIntermediate; + ClassDefNV(SoAIntermediateOld, 2); +}; + +struct RecordLeafOld : public RecordIntermediateOld { + float fLeaf; + ClassDefNV(RecordLeafOld, 2); +}; + +struct SoALeafOld : public SoAIntermediateOld { + ROOT::RVec fLeaf; + ClassDefNV(SoALeafOld, 2); +}; + +struct RecordBaseNew { + float fBase; + float fNew; + ClassDefNV(RecordBaseNew, 2); +}; + +struct SoABaseNew { + ROOT::RVec fBase; + ROOT::RVec fNew; + ClassDefNV(SoABaseNew, 2); +}; + +struct RecordIntermediateNew : public RecordBaseNew { + float fIntermediate; + ClassDefNV(RecordIntermediateNew, 2); +}; + +struct SoAIntermediateNew : public SoABaseNew { + ROOT::RVec fIntermediate; + ClassDefNV(SoAIntermediateNew, 2); +}; + +struct RecordLeafNew : public RecordIntermediateNew { + float fLeaf; + ClassDefNV(RecordLeafNew, 2); +}; + +struct SoALeafNew : public SoAIntermediateNew { + ROOT::RVec fLeaf; + ClassDefNV(SoALeafNew, 2); +}; + +struct RecordNested { + float fInner; + ClassDefNV(RecordNested, 2); +}; + +struct SoANested { + ROOT::RVec fInner; + ClassDefNV(SoANested, 2); +}; + +struct RecordOuter { + float fOuter; + RecordNested fNested; + ClassDefNV(RecordOuter, 2); +}; + +struct SoAOuter { + ROOT::RVec fOuter; + SoANested fNested; + ClassDefNV(SoAOuter, 2); +}; + #endif // ROOT_RNTuple_Test_SoAField diff --git a/tree/ntuple/test/SoAFieldLinkDef.h b/tree/ntuple/test/SoAFieldLinkDef.h index 0af69cec4189a..21d6264af15aa 100644 --- a/tree/ntuple/test/SoAFieldLinkDef.h +++ b/tree/ntuple/test/SoAFieldLinkDef.h @@ -34,4 +34,34 @@ #pragma link C++ options=rntupleSoARecord(RecordDerived) class SoADerivedFail1+; #pragma link C++ options=rntupleSoARecord(RecordDerived) class SoADerivedFail2+; +#pragma link C++ class RecordBaseOld+; +#pragma link C++ class RecordIntermediateOld+; +#pragma link C++ class RecordLeafOld+; +#pragma link C++ options=rntupleSoARecord(RecordBaseOld) class SoABaseOld+; +#pragma link C++ options=rntupleSoARecord(RecordIntermediateOld) class SoAIntermediateOld+; +#pragma link C++ options=rntupleSoARecord(RecordLeafOld) class SoALeafOld+; +#pragma link C++ class RecordBaseNew+; +#pragma link C++ class RecordIntermediateNew+; +#pragma link C++ class RecordLeafNew+; +#pragma link C++ options=rntupleSoARecord(RecordBaseNew) class SoABaseNew+; +#pragma link C++ options=rntupleSoARecord(RecordIntermediateNew) class SoAIntermediateNew+; +#pragma link C++ options=rntupleSoARecord(RecordLeafNew) class SoALeafNew+; + +#pragma read sourceClass = "RecordBaseOld" targetClass = "RecordBaseNew" version = "[1-]" +#pragma read sourceClass = "RecordIntermediateOld" targetClass = "RecordIntermediateNew" version = "[1-]" +#pragma read sourceClass = "RecordLeafOld" targetClass = "RecordLeafNew" version = "[1-]" +#pragma read sourceClass = "SoABaseOld" targetClass = "SoABaseNew" version = "[1-]" +#pragma read sourceClass = "SoAIntermediateOld" targetClass = "SoAIntermediateNew" version = "[1-]" +#pragma read sourceClass = "SoALeafOld" targetClass = "SoALeafNew" version = "[1-]" + +#pragma link C++ class RecordNested+; +#pragma link C++ class RecordOuter+; +#pragma link C++ options=rntupleSoARecord(RecordNested) class SoANested+; +#pragma link C++ options=rntupleSoARecord(RecordOuter) class SoAOuter+; + +#pragma read sourceClass="SoANested" version="[1-]" targetClass="SoANested" source="" target="" \ + code="{ newObj->fInner *= 2.; }" +#pragma read sourceClass="SoAOuter" version="[1-]" targetClass="SoAOuter" source="" target="" \ + code="{ newObj->fOuter *= 4.; }" + #endif // __CLING__ diff --git a/tree/ntuple/test/ntuple_soa.cxx b/tree/ntuple/test/ntuple_soa.cxx index 62b7bf42b4656..4151fbb54a58b 100644 --- a/tree/ntuple/test/ntuple_soa.cxx +++ b/tree/ntuple/test/ntuple_soa.cxx @@ -385,6 +385,7 @@ TEST(RNTuple, SoAFromVector) auto writer = ROOT::RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath()); v->emplace_back(RecordSimple{1.0, 2.0}); + v->emplace_back(RecordSimple{3.0, 4.0}); writer->Fill(); } @@ -392,14 +393,16 @@ TEST(RNTuple, SoAFromVector) auto reader = ROOT::RNTupleReader::Open("ntpl", fileGuard.GetPath()); SoASimple soa; - // Until SoA schema evolution is implemented, the reading the vector as SoA will - try { - reader->GetView("simple", &soa, "SoASimple"); - FAIL() << "reading a vector with a SoA field should fail"; - } catch (const ROOT::RException &e) { - EXPECT_THAT(e.what(), testing::HasSubstr( - "in-memory field simple of type SoASimple is incompatible with on-disk field simple")); - } + std::ostringstream os; + reader->Show(0, os); + // clang-format off + std::string expected{ +R"({ + "simple": [{"fX": 1, "fY": 2}, {"fX": 3, "fY": 4}] +} +)"}; + // clang-format on + EXPECT_EQ(expected, os.str()); } TEST(RNTuple, SoAShow) @@ -576,3 +579,81 @@ R"({ // clang-format on EXPECT_EQ(expected, os.str()); } + +TEST(RNTuple, SoARename) +{ + ROOT::TestSupport::FileRaii fileGuard("test_rntuple_soa_rename.root"); + + { + auto model = ROOT::RNTupleModel::Create(); + + model->AddField(std::make_unique("leaf", "SoALeafOld")); + auto writer = ROOT::RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath()); + + auto leafSoA = writer->GetModel().GetDefaultEntry().GetPtr("leaf"); + leafSoA->fBase = {1.0, 2.0}; + leafSoA->fIntermediate = {3.0, 4.0}; + leafSoA->fLeaf = {5.0, 6.0}; + + writer->Fill(); + } + + auto model = ROOT::RNTupleModel::Create(); + model->AddField(std::make_unique("leaf", "SoALeafNew")); + auto reader = ROOT::RNTupleReader::Open(std::move(model), "ntpl", fileGuard.GetPath()); + + // We cannot use "Show()" because that will reconstruct the original model as a display model, it will not + // use the imposed model. + + auto leafSoA = reader->GetModel().GetDefaultEntry().GetPtr("leaf"); + reader->LoadEntry(0); + + EXPECT_EQ(2u, leafSoA->fBase.size()); + EXPECT_EQ(2u, leafSoA->fNew.size()); + EXPECT_EQ(2u, leafSoA->fIntermediate.size()); + EXPECT_EQ(2u, leafSoA->fLeaf.size()); + EXPECT_FLOAT_EQ(1.0, leafSoA->fBase[0]); + EXPECT_FLOAT_EQ(2.0, leafSoA->fBase[1]); + EXPECT_FLOAT_EQ(0.0, leafSoA->fNew[0]); + EXPECT_FLOAT_EQ(0.0, leafSoA->fNew[1]); + EXPECT_FLOAT_EQ(3.0, leafSoA->fIntermediate[0]); + EXPECT_FLOAT_EQ(4.0, leafSoA->fIntermediate[1]); + EXPECT_FLOAT_EQ(5.0, leafSoA->fLeaf[0]); + EXPECT_FLOAT_EQ(6.0, leafSoA->fLeaf[1]); +} + +TEST(RNTuple, SoAWholeObjectRule) +{ + ROOT::TestSupport::FileRaii fileGuard("test_rntuple_soa_whole_object_rule.root"); + + { + auto model = ROOT::RNTupleModel::Create(); + + model->AddField(std::make_unique("outer", "SoAOuter")); + auto writer = ROOT::RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath()); + + auto outerSoA = writer->GetModel().GetDefaultEntry().GetPtr("outer"); + outerSoA->fNested.fInner = {1.0, 2.0}; + outerSoA->fOuter = {3.0, 4.0}; + + writer->Fill(); + } + + auto reader = ROOT::RNTupleReader::Open("ntpl", fileGuard.GetPath()); + + std::ostringstream os; + reader->Show(0, os); + // clang-format off + std::string expected{ +R"({ + "outer": { + "fOuter": [12, 16], + "fNested": { + "fInner": [2, 4] + } + } +} +)" }; + // clang-format on + EXPECT_EQ(expected, os.str()); +}