Skip to content

DPL Analysis: remove special origins and add From<> version for auto-generated tables #15230

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
aalkin wants to merge 3 commits into
base: dev
Choose a base branch
from
Open

DPL Analysis: remove special origins and add From<> version for auto-generated tables #15230

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
abb56f7
DPL Analysis: use Input/OutputSpec metadata to detect special tables …
aalkin Mar 20, 2026
1688f9c
DPL Analysis: use Input/OutputSpec metadata to detect special tables …
aalkin Mar 20, 2026
a51a5eb
DPL Analysis: remove special origins and add From<> version for auto-…
aalkin Mar 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions Framework/AnalysisSupport/src/AODWriterHelpers.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
#include "Framework/TableConsumer.h"
#include "Framework/DataOutputDirector.h"
#include "Framework/TableTreeHelpers.h"
#include "Framework/Monitoring.h"
#include "Framework/Signpost.h"

#include <Monitoring/Monitoring.h>
Expand All @@ -32,8 +31,6 @@
#include <TMap.h>
#include <TObjString.h>
#include <arrow/table.h>
#include <chrono>
#include <ios>

O2_DECLARE_DYNAMIC_LOG(histogram_registry);

Expand Down Expand Up @@ -157,7 +154,7 @@ AlgorithmSpec AODWriterHelpers::getOutputTTreeWriter(ConfigContext const& ctx)
}

// skip non-AOD refs
if (!DataSpecUtils::partialMatch(*ref.spec, writableAODOrigins)) {
if (!DataSpecUtils::partialMatch(*ref.spec, AODOrigins)) {
continue;
}
startTime = DataRefUtils::getHeader<DataProcessingHeader*>(ref)->startTime;
Expand Down
262 changes: 151 additions & 111 deletions Framework/Core/include/Framework/ASoA.h
View file
Open in desktop

Large diffs are not rendered by default.

34 changes: 20 additions & 14 deletions Framework/Core/include/Framework/AnalysisDataModel.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ DECLARE_SOA_TABLE(BCFlags, "AOD", "BCFLAG", //! flag for tagging UPCs, joinable
bc::Flags);

using BCs = BCs_001; // current version
template <aod::is_origin_hash O>
using BCsFrom = BCs_001From<O>;
using BC = BCs::iterator;

namespace timestamp
Expand All @@ -66,7 +68,7 @@ using BCsWithTimestamps = soa::Join<aod::BCs, aod::Timestamps>;

namespace soa
{
extern template struct JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::BCs, aod::Timestamps>;
extern template struct Join<aod::BCs, aod::Timestamps>;
}
namespace aod
{
Expand Down Expand Up @@ -514,11 +516,11 @@ DECLARE_SOA_TABLE_FULL(StoredTracksIU, "Tracks_IU", "AOD", "TRACK_IU", //! On di
track::Sign<track::Signed1Pt>,
o2::soa::Marker<2>);

DECLARE_SOA_EXTENDED_TABLE(TracksIU, StoredTracksIU, "EXTRACK_IU", 0, //! Track parameters at inner most update (e.g. ITS) as it comes from the tracking
aod::track::Pt,
aod::track::P,
aod::track::Eta,
aod::track::Phi);
DECLARE_SOA_EXTENDED_TABLE_NG(TracksIU, StoredTracksIU, "EXTRACK_IU", 0, //! Track parameters at inner most update (e.g. ITS) as it comes from the tracking
aod::track::Pt,
aod::track::P,
aod::track::Eta,
aod::track::Phi);

DECLARE_SOA_TABLE_FULL(StoredTracksCov, "TracksCov", "AOD", "TRACKCOV", //! On disk version of the TracksCov table at collision vertex
track::SigmaY, track::SigmaZ, track::SigmaSnp, track::SigmaTgl, track::Sigma1Pt,
Expand Down Expand Up @@ -680,9 +682,9 @@ using Run2TrackExtra = Run2TrackExtras::iterator;
} // namespace aod
namespace soa
{
extern template struct soa::JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::Tracks, aod::TracksExtra>;
extern template struct soa::JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::Tracks, aod::TracksCov, aod::TracksExtra>;
extern template struct soa::JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::TracksExtension, aod::StoredTracks>;
extern template struct soa::Join<aod::Tracks, aod::TracksExtra>;
extern template struct soa::Join<aod::Tracks, aod::TracksCov, aod::TracksExtra>;
extern template struct soa::Join<aod::TracksExtension, aod::StoredTracks>;
} // namespace soa
namespace aod
{
Expand Down Expand Up @@ -926,6 +928,8 @@ using MFTTracks = MFTTracks_001;
using StoredMFTTracks = StoredMFTTracks_001;

using MFTTrack = MFTTracks::iterator;
template <aod::is_origin_hash O>
using MFTTracksFrom = MFTTracks_001From<O>;

namespace fwdtrack // Index to MFTtrack column must be defined after table definition.
{
Expand Down Expand Up @@ -1005,7 +1009,7 @@ using MFTTrackCovFwd = MFTTracksCov::iterator;
} // namespace aod
namespace soa
{
extern template struct JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::FwdTracks, aod::FwdTracksCov>;
extern template struct Join<aod::FwdTracks, aod::FwdTracksCov>;
}
namespace aod
{
Expand Down Expand Up @@ -2026,6 +2030,8 @@ DECLARE_SOA_EXTENDED_TABLE(McParticles_001, StoredMcParticles_001, "EXMCPARTICLE
using StoredMcParticles = StoredMcParticles_001;
using McParticles = McParticles_001;
using McParticle = McParticles::iterator;
template <aod::is_origin_hash O>
using McParticlesFrom = McParticles_001From<O>;
} // namespace aod
namespace soa
{
Expand Down Expand Up @@ -2191,11 +2197,11 @@ DECLARE_SOA_INDEX_COLUMN(FDD, fdd); //!
// First entry: Collision
#define INDEX_LIST_RUN2 indices::CollisionId, indices::ZdcId, indices::BCId, indices::FT0Id, indices::FV0AId, indices::FV0CId, indices::FDDId
DECLARE_SOA_INDEX_TABLE_EXCLUSIVE(Run2MatchedExclusive, BCs, "MA_RN2_EX", INDEX_LIST_RUN2); //!
DECLARE_SOA_INDEX_TABLE(Run2MatchedSparse, BCs, "MA_RN2_SP", INDEX_LIST_RUN2); //!
DECLARE_SOA_INDEX_TABLE(Run2MatchedSparse, BCs_001, "MA_RN2_SP", INDEX_LIST_RUN2); //!

#define INDEX_LIST_RUN3 indices::CollisionId, indices::ZdcId, indices::BCId, indices::FT0Id, indices::FV0AId, indices::FDDId
DECLARE_SOA_INDEX_TABLE_EXCLUSIVE(Run3MatchedExclusive, BCs, "MA_RN3_EX", INDEX_LIST_RUN3); //!
DECLARE_SOA_INDEX_TABLE(Run3MatchedSparse, BCs, "MA_RN3_SP", INDEX_LIST_RUN3); //!
DECLARE_SOA_INDEX_TABLE(Run3MatchedSparse, BCs_001, "MA_RN3_SP", INDEX_LIST_RUN3); //!

// First entry: BC
DECLARE_SOA_INDEX_TABLE_EXCLUSIVE(MatchedBCCollisionsExclusive, BCs, "MA_BCCOL_EX", //!
Expand Down Expand Up @@ -2225,8 +2231,8 @@ DECLARE_EQUIVALENT_FOR_INDEX(aod::StoredTracksIU, aod::McTrackLabels);
DECLARE_EQUIVALENT_FOR_INDEX(aod::Collisions, aod::McCollisionLabels);
// Joins with collisions (only for sparse ones)
// NOTE: index table needs to be always last argument
extern template struct JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::Collisions, aod::Run2MatchedSparse>;
extern template struct JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::Collisions, aod::Run3MatchedSparse>;
extern template struct Join<aod::Collisions, aod::Run2MatchedSparse>;
extern template struct Join<aod::Collisions, aod::Run3MatchedSparse>;
} // namespace soa
namespace aod
{
Expand Down
102 changes: 80 additions & 22 deletions Framework/Core/include/Framework/AnalysisHelpers.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ auto makeEmptyTable(const char* name)
}

template <soa::TableRef R>
requires(soa::not_void<typename aod::MetadataTrait<aod::Hash<R.desc_hash>>::metadata>)
auto makeEmptyTable()
{
auto schema = std::make_shared<arrow::Schema>(soa::createFieldsFromColumns(typename aod::MetadataTrait<aod::Hash<R.desc_hash>>::metadata::persistent_columns_t{}));
Expand All @@ -93,6 +94,7 @@ auto makeEmptyTable(const char* name, framework::pack<Cs...> p)
}

template <aod::is_aod_hash D>
requires(soa::not_void<typename aod::MetadataTrait<D>::metadata>)
auto makeEmptyTable(const char* name)
{
auto schema = std::make_shared<arrow::Schema>(soa::createFieldsFromColumns(typename aod::MetadataTrait<D>::metadata::persistent_columns_t{}));
Expand Down Expand Up @@ -216,6 +218,26 @@ inline constexpr auto getSourceSchemas()
}.template operator()<T::sources.size(), T::sources>();
}

template <soa::with_sources_generator T, aod::is_origin_hash O = o2::aod::Hash<"AOD"_h>>
inline constexpr auto getSources()
{
return []<size_t N, std::array<soa::TableRef, N> refs>() {
return []<size_t... Is>(std::index_sequence<Is...>) {
return std::vector{soa::tableRef2ConfigParamSpec<refs[Is]>()...};
}(std::make_index_sequence<N>());
}.template operator()<T::N, T::template generateSources<O>()>();
}

template <soa::with_sources_generator T, aod::is_origin_hash O = o2::aod::Hash<"AOD"_h>>
inline constexpr auto getSourceSchemas()
{
return []<size_t N, std::array<soa::TableRef, N> refs>() {
return []<size_t... Is>(std::index_sequence<Is...>) {
return std::vector{soa::tableRef2Schema<refs[Is]>()...};
}(std::make_index_sequence<N>());
}.template operator()<T::N, T::template generateSources<O>()>();
}

template <soa::with_ccdb_urls T>
inline constexpr auto getCCDBUrls()
{
Expand Down Expand Up @@ -257,7 +279,7 @@ inline constexpr auto getIndexMapping()
using indices = T::index_pack_t;
using Key = T::Key;
[&idx]<size_t... Is>(std::index_sequence<Is...>) mutable {
constexpr auto refs = T::sources;
constexpr auto refs = T::generateSources();
([&idx]<TableRef ref, typename C>() mutable {
constexpr auto pos = o2::aod::MetadataTrait<o2::aod::Hash<ref.desc_hash>>::metadata::template getIndexPosToKey<Key>();
if constexpr (pos == -1) {
Expand All @@ -272,6 +294,26 @@ inline constexpr auto getIndexMapping()
return idx;
}

template <soa::with_sources_generator T, aod::is_origin_hash O = o2::aod::Hash<"AOD"_h>>
constexpr auto getInputMetadata() -> std::vector<framework::ConfigParamSpec>
{
std::vector<framework::ConfigParamSpec> inputMetadata;

auto inputSources = getSources<T, O>();
std::sort(inputSources.begin(), inputSources.end(), [](framework::ConfigParamSpec const& a, framework::ConfigParamSpec const& b) { return a.name < b.name; });
auto last = std::unique(inputSources.begin(), inputSources.end(), [](framework::ConfigParamSpec const& a, framework::ConfigParamSpec const& b) { return a.name == b.name; });
inputSources.erase(last, inputSources.end());
inputMetadata.insert(inputMetadata.end(), inputSources.begin(), inputSources.end());

auto inputSchemas = getSourceSchemas<T, O>();
std::sort(inputSchemas.begin(), inputSchemas.end(), [](framework::ConfigParamSpec const& a, framework::ConfigParamSpec const& b) { return a.name < b.name; });
last = std::unique(inputSchemas.begin(), inputSchemas.end(), [](framework::ConfigParamSpec const& a, framework::ConfigParamSpec const& b) { return a.name == b.name; });
inputSchemas.erase(last, inputSchemas.end());
inputMetadata.insert(inputMetadata.end(), inputSchemas.begin(), inputSchemas.end());

return inputMetadata;
}

template <soa::with_sources T>
constexpr auto getInputMetadata() -> std::vector<framework::ConfigParamSpec>
{
Expand All @@ -293,7 +335,7 @@ constexpr auto getInputMetadata() -> std::vector<framework::ConfigParamSpec>
}

template <typename T>
requires(!soa::with_sources<T>)
requires(!(soa::with_sources<T> || soa::with_sources_generator<T>))
constexpr auto getInputMetadata() -> std::vector<framework::ConfigParamSpec>
{
return {};
Expand Down Expand Up @@ -358,14 +400,19 @@ template <TableRef R>
constexpr auto tableRef2InputSpec()
{
std::vector<framework::ConfigParamSpec> metadata;
auto m = getInputMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>();
metadata.insert(metadata.end(), m.begin(), m.end());
auto ccdbMetadata = getCCDBMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>();
metadata.insert(metadata.end(), ccdbMetadata.begin(), ccdbMetadata.end());
auto p = getExpressionMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>();
metadata.insert(metadata.end(), p.begin(), p.end());
auto idx = getIndexMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>();
metadata.insert(metadata.end(), idx.begin(), idx.end());
std::vector<framework::ConfigParamSpec> sources;
if constexpr (soa::with_sources<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>) {
sources = getInputMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>();
} else if constexpr (soa::with_sources_generator<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>) {
sources = getInputMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata, o2::aod::Hash<R.origin_hash>>();
}
metadata.insert(metadata.end(), sources.begin(), sources.end());
auto ccdbURLs = getCCDBMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>();
metadata.insert(metadata.end(), ccdbURLs.begin(), ccdbURLs.end());
auto expressions = getExpressionMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>();
metadata.insert(metadata.end(), expressions.begin(), expressions.end());
auto indices = getIndexMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>();
metadata.insert(metadata.end(), indices.begin(), indices.end());
if constexpr (!soa::with_ccdb_urls<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>) {
metadata.emplace_back(framework::ConfigParamSpec{"schema", framework::VariantType::String, framework::serializeSchema(o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata::getSchema()), {"\"\""}});
}
Expand All @@ -382,11 +429,22 @@ constexpr auto tableRef2InputSpec()
template <TableRef R>
constexpr auto tableRef2OutputSpec()
{
std::vector<framework::ConfigParamSpec> metadata;
using md = typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata;
if constexpr (soa::with_ccdb_urls<md>) {
metadata.emplace_back("ccdb:", framework::VariantType::Bool, true, framework::ConfigParamSpec::HelpString{"\"\""});
} else if constexpr (soa::with_expression_pack<md>) {
metadata.emplace_back("projectors", framework::VariantType::Bool, true, framework::ConfigParamSpec::HelpString{"\"\""});
} else if constexpr (soa::with_index_pack<md>) {
metadata.emplace_back("index-records", framework::VariantType::Bool, true, framework::ConfigParamSpec::HelpString{"\"\""});
}
return framework::OutputSpec{
framework::OutputLabel{o2::aod::label<R>()},
o2::aod::origin<R>(),
o2::aod::description(o2::aod::signature<R>()),
R.version};
R.version,
framework::Lifetime::Timeframe,
metadata};
}

template <TableRef R>
Expand Down Expand Up @@ -504,14 +562,14 @@ struct OutputForTable {
using table_t = decltype(typeWithRef<T>());
using metadata = aod::MetadataTrait<o2::aod::Hash<table_t::ref.desc_hash>>::metadata;

static OutputSpec const spec()
static constexpr auto spec()
{
return OutputSpec{OutputLabel{aod::label<table_t::ref>()}, o2::aod::origin<table_t::ref>(), o2::aod::description(o2::aod::signature<table_t::ref>()), table_t::ref.version};
return soa::tableRef2OutputSpec<table_t::ref>();
}

static OutputRef ref()
static constexpr auto ref()
{
return OutputRef{aod::label<table_t::ref>(), table_t::ref.version};
return soa::tableRef2OutputRef<table_t::ref>();
}
};

Expand Down Expand Up @@ -544,7 +602,7 @@ concept is_produces_group = std::derived_from<T, ProducesGroup>;
template <soa::is_metadata M, soa::TableRef Ref>
struct TableTransform {
using metadata = M;
constexpr static auto sources = M::sources;
constexpr static auto sources = M::template generateSources<o2::aod::Hash<Ref.origin_hash>>();

template <soa::TableRef R>
static auto base_spec()
Expand Down Expand Up @@ -578,23 +636,23 @@ struct TableTransform {
/// This helper struct allows you to declare extended tables which should be
/// created by the task (as opposed to those pre-defined by data model)
template <typename T>
concept is_spawnable = soa::has_metadata<aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>> && soa::has_extension<typename aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>::metadata>;
concept is_spawnable = soa::has_metadata<aod::MetadataTrait<o2::aod::Hash<T::originals[T::originals.size() - 1].desc_hash>>> && soa::has_extension<typename aod::MetadataTrait<o2::aod::Hash<T::originals[T::originals.size() - 1].desc_hash>>::metadata>;

template <typename T>
concept is_dynamically_spawnable = soa::has_metadata<aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>> && soa::has_configurable_extension<typename aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>::metadata>;
concept is_dynamically_spawnable = soa::has_metadata<aod::MetadataTrait<o2::aod::Hash<T::originals[T::originals.size() - 1].desc_hash>>> && soa::has_configurable_extension<typename aod::MetadataTrait<o2::aod::Hash<T::originals[T::originals.size() - 1].desc_hash>>::metadata>;

template <is_spawnable T>
constexpr auto transformBase()
{
using metadata = typename aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>::metadata;
return TableTransform<metadata, metadata::extension_table_t::ref>{};
using metadata = typename aod::MetadataTrait<o2::aod::Hash<T::originals[T::originals.size() - 1].desc_hash>>::metadata;
return TableTransform<metadata, metadata::template extension_table_t_from<o2::aod::Hash<T::originals[T::originals.size() - 1].origin_hash>>::ref>{};
}

template <is_spawnable T>
struct Spawns : decltype(transformBase<T>()) {
using spawnable_t = T;
using metadata = decltype(transformBase<T>())::metadata;
using extension_t = typename metadata::extension_table_t;
using extension_t = typename metadata::template extension_table_t_from<o2::aod::Hash<T::originals[T::originals.size() - 1].origin_hash>>;
using expression_pack_t = typename metadata::expression_pack_t;
static constexpr size_t N = framework::pack_size(expression_pack_t{});

Expand Down Expand Up @@ -644,7 +702,7 @@ struct Defines : decltype(transformBase<T>()) {
static constexpr bool delayed = DELAYED;
using spawnable_t = T;
using metadata = decltype(transformBase<T>())::metadata;
using extension_t = typename metadata::extension_table_t;
using extension_t = typename metadata::template extension_table_t_from<o2::aod::Hash<T::originals[T::originals.size() - 1].origin_hash>>;
using placeholders_pack_t = typename metadata::placeholders_pack_t;
static constexpr size_t N = framework::pack_size(placeholders_pack_t{});

Expand Down
Loading
Loading