From e22d9ae872696e794f53ff6f64bbfbdcc341ad87 Mon Sep 17 00:00:00 2001 From: Jakob Blomer Date: Mon, 27 Jul 2026 23:23:59 +0200 Subject: [PATCH 1/5] [ntuple] read SoA field from different on-disk version --- tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx | 2 ++ tree/ntuple/src/RFieldMeta.cxx | 5 +++++ tree/ntuple/test/ntuple_evolution_soa.cxx | 0 3 files changed, 7 insertions(+) create mode 100644 tree/ntuple/test/ntuple_evolution_soa.cxx diff --git a/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx b/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx index 1f7ff8a16089c..bd5c2b6a35ee9 100644 --- a/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx +++ b/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx @@ -114,6 +114,8 @@ protected: void CommitClusterImpl() final { fNWritten = 0; } + void ReconcileOnDiskField(const RNTupleDescriptor &desc) final; + public: RSoAField(std::string_view fieldName, std::string_view className); RSoAField(RSoAField &&other) = default; diff --git a/tree/ntuple/src/RFieldMeta.cxx b/tree/ntuple/src/RFieldMeta.cxx index 9f4cf06be45e5..fb61f806396b5 100644 --- a/tree/ntuple/src/RFieldMeta.cxx +++ b/tree/ntuple/src/RFieldMeta.cxx @@ -987,6 +987,11 @@ void ROOT::Experimental::RSoAField::ReadGlobalImpl(ROOT::NTupleSize_t globalInde } } +void ROOT::Experimental::RSoAField::ReconcileOnDiskField(const RNTupleDescriptor &desc) +{ + EnsureMatchingOnDiskField(desc, kDiffTypeVersion).ThrowOnError(); +} + void ROOT::Experimental::RSoAField::ConstructValue(void *where) const { fSoAClass->New(where); diff --git a/tree/ntuple/test/ntuple_evolution_soa.cxx b/tree/ntuple/test/ntuple_evolution_soa.cxx new file mode 100644 index 0000000000000..e69de29bb2d1d From d6bd75d4dec68db7e5e8343f9479e11963548db8 Mon Sep 17 00:00:00 2001 From: Jakob Blomer Date: Mon, 27 Jul 2026 23:24:51 +0200 Subject: [PATCH 2/5] [ntuple] test SoA member removal schema evolution --- tree/ntuple/test/CMakeLists.txt | 1 + tree/ntuple/test/ntuple_evolution_soa.cxx | 121 ++++++++++++++++++++++ 2 files changed, 122 insertions(+) diff --git a/tree/ntuple/test/CMakeLists.txt b/tree/ntuple/test/CMakeLists.txt index 4fd5fadfed5f8..a781f97cf62db 100644 --- a/tree/ntuple/test/CMakeLists.txt +++ b/tree/ntuple/test/CMakeLists.txt @@ -42,6 +42,7 @@ ROOT_GENERATE_DICTIONARY(STLContainerEvolutionDict ${CMAKE_CURRENT_SOURCE_DIR}/S if(NOT MSVC) # These unit tests rely on fork(), which is not available on Windows. ROOT_ADD_GTEST(ntuple_evolution_shape ntuple_evolution_shape.cxx LIBRARIES ROOTNTuple) + ROOT_ADD_GTEST(ntuple_evolution_soa ntuple_evolution_soa.cxx LIBRARIES ROOTNTuple) ROOT_ADD_GTEST(ntuple_emulated ntuple_emulated.cxx LIBRARIES ROOTNTuple) endif() ROOT_ADD_GTEST(ntuple_field_name ntuple_field_name.cxx LIBRARIES ROOTNTuple) diff --git a/tree/ntuple/test/ntuple_evolution_soa.cxx b/tree/ntuple/test/ntuple_evolution_soa.cxx index e69de29bb2d1d..fbd82849b599b 100644 --- a/tree/ntuple/test/ntuple_evolution_soa.cxx +++ b/tree/ntuple/test/ntuple_evolution_soa.cxx @@ -0,0 +1,121 @@ +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include "gtest/gtest.h" +#include "ntuple_fork.hxx" + +namespace { + +void EvaluateIntImpl(const char *expression, int *value) +{ + auto interpreterValue = gInterpreter->MakeInterpreterValue(); + ASSERT_TRUE(gInterpreter->Evaluate(expression, *interpreterValue)); + *value = interpreterValue->GetAsLong(); +} + +#define EXPECT_EVALUATE_EQ(expression, expected) \ + do { \ + int _value; \ + EvaluateIntImpl(expression, &_value); \ + if (::testing::Test::HasFatalFailure()) \ + return; \ + EXPECT_EQ(expected, _value); \ + } while (0) + +void MakeSoALink(const std::string &recordName, const std::string &soaName) +{ + auto cl = TClass::GetClass(soaName.c_str()); + cl->CreateAttributeMap(); + cl->GetAttributeMap()->AddProperty("rntuple.SoARecord", recordName.c_str()); +} + +} // namespace + +TEST(RNTupleEvolutionSoA, RemovedMember) +{ + ROOT::TestSupport::FileRaii fileGuard("test_ntuple_evolution_soa_removed_member.root"); + + ExecInFork([&] { + // The child process writes the file and exits, but the file must be preserved to be read by the parent. + fileGuard.PreserveFile(); + + ROOT::TestSupport::CheckDiagsRAII diagRAII; + diagRAII.requiredDiag(kWarning, "[ROOT.NTuple]", "The SoA field is experimental and still under development.", + true /* matchFullMessage */); + + ASSERT_TRUE(gInterpreter->Declare(R"( +struct RemovedMemberRecord { + int fInt1; + int fInt2; + int fInt3; + ClassDefNV(RemovedMemberRecord, 2) +}; +struct RemovedMemberSoA { + ROOT::RVec fInt1; + ROOT::RVec fInt2; + ROOT::RVec fInt3; + ClassDefNV(RemovedMemberSoA, 2) +}; +)")); + MakeSoALink("RemovedMemberRecord", "RemovedMemberSoA"); + + auto model = ROOT::RNTupleModel::Create(); + model->AddField(ROOT::RFieldBase::Create("f", "RemovedMemberSoA").Unwrap()); + + auto writer = ROOT::RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath()); + writer->Fill(); + + void *ptr = writer->GetModel().GetDefaultEntry().GetPtr("f").get(); + DeclarePointer("RemovedMemberSoA", "ptrRemovedMember", ptr); + ProcessLine("ptrRemovedMember->fInt1 = {11, 12};"); + ProcessLine("ptrRemovedMember->fInt2 = {13, 14};"); + ProcessLine("ptrRemovedMember->fInt3 = {15, 16};"); + writer->Fill(); + + // Reset / close the writer and flush the file. + writer.reset(); + }); + + ASSERT_TRUE(gInterpreter->Declare(R"( +struct RemovedMemberRecord { + int fInt1; + int fInt3; + ClassDefNV(RemovedMemberRecord, 3) +}; +struct RemovedMemberSoA { + ROOT::RVec fInt1; + ROOT::RVec fInt3; + ClassDefNV(RemovedMemberSoA, 3) +}; +)")); + MakeSoALink("RemovedMemberRecord", "RemovedMemberSoA"); + + ROOT::TestSupport::CheckDiagsRAII diagRAII; + diagRAII.requiredDiag(kWarning, "[ROOT.NTuple]", "The SoA field is experimental and still under development.", + true /* matchFullMessage */); + + auto reader = ROOT::RNTupleReader::Open("ntpl", fileGuard.GetPath()); + ASSERT_EQ(2, reader->GetNEntries()); + + void *ptr = reader->GetModel().GetDefaultEntry().GetPtr("f").get(); + DeclarePointer("RemovedMemberSoA", "ptrRemovedMember", ptr); + + reader->LoadEntry(0); + EXPECT_EVALUATE_EQ("ptrRemovedMember->fInt1.size()", 0); + EXPECT_EVALUATE_EQ("ptrRemovedMember->fInt3.size()", 0); + + reader->LoadEntry(1); + EXPECT_EVALUATE_EQ("ptrRemovedMember->fInt1[0]", 11); + EXPECT_EVALUATE_EQ("ptrRemovedMember->fInt1[1]", 12); + EXPECT_EVALUATE_EQ("ptrRemovedMember->fInt3[0]", 15); + EXPECT_EVALUATE_EQ("ptrRemovedMember->fInt3[1]", 16); +} From 11c51cf9a1cf518f1b499ee72c730be273dda602 Mon Sep 17 00:00:00 2001 From: Jakob Blomer Date: Tue, 28 Jul 2026 12:51:58 +0200 Subject: [PATCH 3/5] [ntuple] test SoA type change schema evolution --- tree/ntuple/test/ntuple_evolution_soa.cxx | 64 +++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/tree/ntuple/test/ntuple_evolution_soa.cxx b/tree/ntuple/test/ntuple_evolution_soa.cxx index fbd82849b599b..2cf3da812da98 100644 --- a/tree/ntuple/test/ntuple_evolution_soa.cxx +++ b/tree/ntuple/test/ntuple_evolution_soa.cxx @@ -119,3 +119,67 @@ struct RemovedMemberSoA { EXPECT_EVALUATE_EQ("ptrRemovedMember->fInt3[0]", 15); EXPECT_EVALUATE_EQ("ptrRemovedMember->fInt3[1]", 16); } + +TEST(RNTupleEvolutionSoA, TypeChange) +{ + ROOT::TestSupport::FileRaii fileGuard("test_ntuple_evolution_soa_type_change.root"); + + ExecInFork([&] { + // The child process writes the file and exits, but the file must be preserved to be read by the parent. + fileGuard.PreserveFile(); + + ASSERT_TRUE(gInterpreter->Declare(R"( +struct TypeChangeRecord { + bool fInt1; + long long int fInt2; + ClassDefNV(TypeChangeRecord, 2) +}; +struct TypeChangeSoA { + ROOT::RVec fInt1; + ROOT::RVec fInt2; + ClassDefNV(TypeChangeSoA, 2) +}; +)")); + MakeSoALink("TypeChangeRecord", "TypeChangeSoA"); + + auto model = ROOT::RNTupleModel::Create(); + model->AddField(ROOT::RFieldBase::Create("f", "TypeChangeSoA").Unwrap()); + + auto writer = ROOT::RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath()); + + void *ptr = writer->GetModel().GetDefaultEntry().GetPtr("f").get(); + DeclarePointer("TypeChangeSoA", "ptrTypeChange", ptr); + ProcessLine("ptrTypeChange->fInt1 = {true, false};"); + ProcessLine("ptrTypeChange->fInt2 = {137, 138};"); + writer->Fill(); + + // Reset / close the writer and flush the file. + writer.reset(); + }); + + ASSERT_TRUE(gInterpreter->Declare(R"( +struct TypeChangeRecord { + int fInt1; + int fInt2; + ClassDefNV(TypeChangeRecord, 3) +}; +struct TypeChangeSoA { + ROOT::RVec fInt1; + ROOT::RVec fInt2; + ClassDefNV(TypeChangeSoA, 3) +}; +)")); + MakeSoALink("TypeChangeRecord", "TypeChangeSoA"); + + auto reader = ROOT::RNTupleReader::Open("ntpl", fileGuard.GetPath()); + ASSERT_EQ(1, reader->GetNEntries()); + + void *ptr = reader->GetModel().GetDefaultEntry().GetPtr("f").get(); + DeclarePointer("TypeChangeSoA", "ptrTypeChange", ptr); + + reader->LoadEntry(0); + EXPECT_EVALUATE_EQ("ptrTypeChange->fInt1[0]", 1); + EXPECT_EVALUATE_EQ("ptrTypeChange->fInt1[1]", 0); + EXPECT_EVALUATE_EQ("ptrTypeChange->fInt2[0]", 137); + EXPECT_EVALUATE_EQ("ptrTypeChange->fInt2[1]", 138); +} From c3e395eaf6d1bdda9ba455d49035d9d382c01438 Mon Sep 17 00:00:00 2001 From: Jakob Blomer Date: Thu, 30 Jul 2026 06:07:14 +0200 Subject: [PATCH 4/5] [ntuple] explicitly zero-initialize added SoA members --- tree/ntuple/src/RFieldMeta.cxx | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/tree/ntuple/src/RFieldMeta.cxx b/tree/ntuple/src/RFieldMeta.cxx index fb61f806396b5..e67fc54202a52 100644 --- a/tree/ntuple/src/RFieldMeta.cxx +++ b/tree/ntuple/src/RFieldMeta.cxx @@ -977,11 +977,25 @@ void ROOT::Experimental::RSoAField::ReadGlobalImpl(ROOT::NTupleSize_t globalInde void *rvecPtr = static_cast(to) + fSoAMemberOffsets[i]; auto begin = ROOT::RRVecField::ResizeRVec(rvecPtr, N, memberSize, memberField, fRecordMemberDeleters[i].get()); - if (memberField->IsSimple() && N) { + if (N == 0) + continue; + + if (memberField->IsSimple()) { GetPrincipalColumnOf(*memberField)->ReadV(collectionStart, N, begin); } else { - for (std::size_t j = 0; j < N; ++j) { - CallReadOn(*memberField, collectionStart + j, begin + (j * memberSize)); + if (memberField->IsArtificial()) { + // Other artificial fields simply don't read at all. This does not work here because then + // the vector elements of trivial types would be left uninitialized (complex types explicitly call + // the constructor on vector resize). Thus we explicitly default-initialize trivial types. + // Note that this causes a subtle difference in behavior: if the added member is default-initialized to + // a non-zero value, this will be forgotten in the SoA layout. + if (memberField->GetTraits() & kTraitTriviallyConstructible) { + std::memset(begin, 0, N * memberSize); + } + } else { + for (std::size_t j = 0; j < N; ++j) { + CallReadOn(*memberField, collectionStart + j, begin + (j * memberSize)); + } } } } From 6e665d87113a5986043558cb0707be2da889d416 Mon Sep 17 00:00:00 2001 From: Jakob Blomer Date: Tue, 28 Jul 2026 23:21:55 +0200 Subject: [PATCH 5/5] [ntuple] test SoA member addition schema evolution --- tree/ntuple/test/ntuple_evolution_soa.cxx | 68 +++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/tree/ntuple/test/ntuple_evolution_soa.cxx b/tree/ntuple/test/ntuple_evolution_soa.cxx index 2cf3da812da98..6ffc0cd735b3c 100644 --- a/tree/ntuple/test/ntuple_evolution_soa.cxx +++ b/tree/ntuple/test/ntuple_evolution_soa.cxx @@ -183,3 +183,71 @@ struct TypeChangeSoA { EXPECT_EVALUATE_EQ("ptrTypeChange->fInt2[0]", 137); EXPECT_EVALUATE_EQ("ptrTypeChange->fInt2[1]", 138); } + +TEST(RNTupleEvolutionSoA, AddedMember) +{ + ROOT::TestSupport::FileRaii fileGuard("test_ntuple_evolution_soa_added_member.root"); + + ExecInFork([&] { + // The child process writes the file and exits, but the file must be preserved to be read by the parent. + fileGuard.PreserveFile(); + + ASSERT_TRUE(gInterpreter->Declare(R"( +struct AddedMemberRecord { + int fInt1; + ClassDefNV(AddedMemberRecord, 2) +}; +struct AddedMemberSoA { + ROOT::RVec fInt1; + ClassDefNV(AddedMemberSoA, 2) +}; +)")); + MakeSoALink("AddedMemberRecord", "AddedMemberSoA"); + + auto model = ROOT::RNTupleModel::Create(); + model->AddField(ROOT::RFieldBase::Create("f", "AddedMemberSoA").Unwrap()); + + auto writer = ROOT::RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath()); + + void *ptr = writer->GetModel().GetDefaultEntry().GetPtr("f").get(); + DeclarePointer("AddedMemberSoA", "ptrAddedMember", ptr); + ProcessLine("for (int i = 0; i < 1000; ++i) ptrAddedMember->fInt1.push_back(137);"); + writer->Fill(); + + // Reset / close the writer and flush the file. + writer.reset(); + }); + + ASSERT_TRUE(gInterpreter->Declare(R"( +struct AddedMemberRecord { + int fInt1; + int fInt2; + std::string fStr; + ClassDefNV(AddedMemberRecord, 3) +}; +struct AddedMemberSoA { + ROOT::RVec fInt1; + ROOT::RVec fInt2; + ROOT::RVec fStr; + ClassDefNV(AddedMemberSoA, 3) +}; +)")); + MakeSoALink("AddedMemberRecord", "AddedMemberSoA"); + + auto reader = ROOT::RNTupleReader::Open("ntpl", fileGuard.GetPath()); + ASSERT_EQ(1, reader->GetNEntries()); + + void *ptr = reader->GetModel().GetDefaultEntry().GetPtr("f").get(); + DeclarePointer("AddedMemberSoA", "ptrAddedMember", ptr); + + reader->LoadEntry(0); + EXPECT_EVALUATE_EQ("ptrAddedMember->fInt1.size()", 1000); + EXPECT_EVALUATE_EQ("ptrAddedMember->fInt1[0]", 137); + EXPECT_EVALUATE_EQ("ptrAddedMember->fInt1[999]", 137); + EXPECT_EVALUATE_EQ("ptrAddedMember->fInt2.size()", 1000); + EXPECT_EVALUATE_EQ("ptrAddedMember->fInt2[0]", 0); + EXPECT_EVALUATE_EQ("ptrAddedMember->fInt2[999]", 0); + EXPECT_EVALUATE_EQ("ptrAddedMember->fStr.size()", 1000); + EXPECT_EVALUATE_EQ("ptrAddedMember->fStr[0].size()", 0); + EXPECT_EVALUATE_EQ("ptrAddedMember->fStr[999].size()", 0); +}