Send reconstruction ConfigurableParams to AOD metadata writer

Common/Utils/include/CommonUtils/ConfigurableParam.h

@@ -407,6 +407,8 @@ class ConfigurableParam
   // writes a human readable INI or JSON file depending on the extension
   static void write(std::string const& filename, std::string const& keyOnly = "");
 
+  static std::string asJSON(std::string const& keyOnly = "");
+
   // can be used instead of using API on concrete child classes
   template <typename T>
   static T getValueAs(std::string key)
@@ -494,6 +496,9 @@ class ConfigurableParam
   // be updated, absence of data for any of requested params will lead to fatal
   static void updateFromFile(std::string const&, std::string const& paramsList = "", bool unchangedOnly = false);
 
+  // update from a JSON string with the same filtering semantics as updateFromFile
+  static void updateFromJSONString(std::string const&, std::string const& paramsList = "", bool unchangedOnly = false);
+
   // interface for use from the CCDB API; allows to sync objects read from CCDB with the information
   // stored in the registry; modifies given object as well as registry
   virtual void syncCCDBandRegistry(void* obj) = 0;

Common/Utils/src/ConfigurableParam.cxx

@@ -37,6 +37,7 @@
 #endif
 #include <cassert>
 #include <iostream>
+#include <sstream>
 #include <string>
 #include <fairlogger/Logger.h>
 #include <typeindex>
@@ -754,6 +755,29 @@ void ConfigurableParam::writeJSON(std::string const& filename, std::string const
 
 // ------------------------------------------------------------------
 
+std::string ConfigurableParam::asJSON(std::string const& keyOnly)
+{
+  initPropertyTree(); // update the boost tree before writing
+  std::ostringstream os;
+  if (!keyOnly.empty()) { // write ini for selected key only
+    try {
+      boost::property_tree::ptree kTree;
+      auto keys = o2::utils::Str::tokenize(keyOnly, " ,;", true, true);
+      for (const auto& k : keys) {
+        kTree.add_child(k, sPtree->get_child(k));
+      }
+      boost::property_tree::write_json(os, kTree);
+    } catch (const boost::property_tree::ptree_bad_path& err) {
+      LOG(fatal) << "non-existing key " << keyOnly << " provided to writeJSON";
+    }
+  } else {
+    boost::property_tree::write_json(os, *sPtree);
+  }
+  return os.str();
+}
+
+// ------------------------------------------------------------------
+
 void ConfigurableParam::initPropertyTree()
 {
   sPtree->clear();
@@ -870,26 +894,10 @@ void ConfigurableParam::printAllRegisteredParamNames()
 
 // ------------------------------------------------------------------
 
-// Update the storage map of params from the given configuration file.
-// It can be in JSON or INI format.
-// If nonempty comma-separated paramsList is provided, only those params will
-// be updated, absence of data for any of requested params will lead to fatal
-// If unchangedOnly is true, then only those parameters whose provenance is kCODE will be updated
-// (to allow preference of run-time settings)
-void ConfigurableParam::updateFromFile(std::string const& configFile, std::string const& paramsList, bool unchangedOnly)
+namespace
+{
+void updateFromPropertyTree(boost::property_tree::ptree const& pt, std::string const& source, std::string const& paramsList, bool unchangedOnly)
 {
-  if (!sIsFullyInitialized) {
-    initialize();
-  }
-
-  auto cfgfile = o2::utils::Str::trim_copy(configFile);
-
-  if (cfgfile.length() == 0) {
-    return;
-  }
-
-  boost::property_tree::ptree pt = ConfigurableParamReaders::readConfigFile(cfgfile);
-
   std::vector<std::pair<std::string, std::string>> keyValPairs;
   auto request = o2::utils::Str::tokenize(paramsList, ',', true);
   std::unordered_map<std::string, int> requestMap;
@@ -913,7 +921,7 @@ void ConfigurableParam::updateFromFile(std::string const& configFile, std::strin
         auto name = subKey.first;
         auto value = subKey.second.get_value<std::string>();
         std::string key = mainKey + "." + name;
-        if (!unchangedOnly || getProvenance(key) == kCODE) {
+        if (!unchangedOnly || ConfigurableParam::getProvenance(key) == ConfigurableParam::kCODE) {
           std::pair<std::string, std::string> pair = std::make_pair(key, o2::utils::Str::trim_copy(value));
           keyValPairs.push_back(pair);
         }
@@ -928,16 +936,62 @@ void ConfigurableParam::updateFromFile(std::string const& configFile, std::strin
   // make sure all requested params were retrieved
   for (const auto& req : requestMap) {
     if (req.second == 0) {
-      throw std::runtime_error(fmt::format("Param {:s} was not found in {:s}", req.first, configFile));
+      throw std::runtime_error(fmt::format("Param {:s} was not found in {:s}", req.first, source));
     }
   }
 
   try {
-    setValues(keyValPairs);
+    ConfigurableParam::setValues(keyValPairs);
   } catch (std::exception const& error) {
     LOG(error) << "Error while setting values " << error.what();
   }
 }
+} // namespace
+
+// Update the storage map of params from the given configuration file.
+// It can be in JSON or INI format.
+// If nonempty comma-separated paramsList is provided, only those params will
+// be updated, absence of data for any of requested params will lead to fatal
+// If unchangedOnly is true, then only those parameters whose provenance is kCODE will be updated
+// (to allow preference of run-time settings)
+void ConfigurableParam::updateFromFile(std::string const& configFile, std::string const& paramsList, bool unchangedOnly)
+{
+  if (!sIsFullyInitialized) {
+    initialize();
+  }
+
+  auto cfgfile = o2::utils::Str::trim_copy(configFile);
+
+  if (cfgfile.length() == 0) {
+    return;
+  }
+
+  updateFromPropertyTree(ConfigurableParamReaders::readConfigFile(cfgfile), configFile, paramsList, unchangedOnly);
+}
+
+// ------------------------------------------------------------------
+
+void ConfigurableParam::updateFromJSONString(std::string const& configJSON, std::string const& paramsList, bool unchangedOnly)
+{
+  if (!sIsFullyInitialized) {
+    initialize();
+  }
+
+  auto json = o2::utils::Str::trim_copy(configJSON);
+  if (json.length() == 0) {
+    return;
+  }
+
+  boost::property_tree::ptree pt;
+  std::istringstream input(json);
+  try {
+    boost::property_tree::read_json(input, pt);
+  } catch (const boost::property_tree::ptree_error& e) {
+    LOG(fatal) << "Failed to read JSON config string (" << e.what() << ")";
+  }
+
+  updateFromPropertyTree(pt, "provided JSON string", paramsList, unchangedOnly);
+}
 
 // ------------------------------------------------------------------
 // ------------------------------------------------------------------

Common/Utils/test/testConfigurableParam.cxx

@@ -151,6 +151,42 @@ BOOST_AUTO_TEST_CASE(ConfigurableParam_FileIO_Json)
   std::remove(testFileName.c_str());
 }
 
+BOOST_AUTO_TEST_CASE(ConfigurableParam_JSONString_UnchangedOnly)
+{
+  ConfigurableParam::setValue("TestParam.ulValue", "2");
+  ConfigurableParam::setProvenance("TestParam", "lValue", ConfigurableParam::kCODE);
+  ConfigurableParam::setProvenance("TestParam", "ulValue", ConfigurableParam::kRT);
+  ConfigurableParam::updateFromJSONString(R"json({"TestParam":{"lValue":"77","ulValue":"88"}})json", "TestParam", true);
+
+  BOOST_CHECK_EQUAL(TestParam::Instance().lValue, 77);
+  BOOST_CHECK_EQUAL(TestParam::Instance().ulValue, 2);
+}
+
+BOOST_AUTO_TEST_CASE(ConfigurableParam_JSONString_FromAsJSON)
+{
+  ConfigurableParam::setValue("TestParam.iValue", "321");
+  ConfigurableParam::setValue("TestParam.sValue", "json-source");
+  ConfigurableParam::setValues({{"TestParam.map", "{7:8,9:10}"}});
+  const auto mapBefore = TestParam::Instance().map;
+  const auto json = ConfigurableParam::asJSON("TestParam");
+
+  ConfigurableParam::setValue("TestParam.iValue", "999");
+  ConfigurableParam::setValue("TestParam.sValue", "json-modified");
+  ConfigurableParam::setValues({{"TestParam.map", "{1:2}"}});
+  ConfigurableParam::updateFromJSONString(json, "TestParam");
+
+  BOOST_CHECK_EQUAL(TestParam::Instance().iValue, 321);
+  BOOST_CHECK_EQUAL(TestParam::Instance().sValue, "json-source");
+  BOOST_CHECK_EQUAL(TestParam::Instance().map.size(), mapBefore.size());
+  BOOST_CHECK_EQUAL(TestParam::Instance().map.at(7), mapBefore.at(7));
+  BOOST_CHECK_EQUAL(TestParam::Instance().map.at(9), mapBefore.at(9));
+}
+
+BOOST_AUTO_TEST_CASE(ConfigurableParam_JSONString_ParamsListMissing)
+{
+  BOOST_CHECK_THROW(ConfigurableParam::updateFromJSONString(ConfigurableParam::asJSON("TestParam"), "MissingParam"), std::runtime_error);
+}
+
 BOOST_AUTO_TEST_CASE(ConfigurableParam_FileIO_ROOT)
 {
   // test for root file serialization

Detectors/GlobalTrackingWorkflow/src/CosmicsMatchingSpec.cxx

@@ -11,10 +11,13 @@
 
 /// @file   CosmicsMatchingSpec.cxx
 
+#include <TMap.h>
+#include <TObjString.h>
 #include <vector>
 #include <string>
 #include "TStopwatch.h"
 #include "GlobalTracking/MatchCosmics.h"
+#include "GlobalTracking/MatchCosmicsParams.h"
 #include "DataFormatsITSMFT/TopologyDictionary.h"
 #include "DataFormatsTPC/Constants.h"
 #include "ReconstructionDataFormats/GlobalTrackID.h"
@@ -104,6 +107,18 @@ void CosmicsMatchingSpec::run(ProcessingContext& pc)
   if (mUseMC) {
     pc.outputs().snapshot(Output{"GLO", "COSMICTRC_MC", 0}, mMatching.getCosmicTracksLbl());
   }
+  static bool first = true;
+  if (first) {
+    first = false;
+    const auto& conf = MatchCosmicsParams::Instance();
+    if (pc.services().get<const o2::framework::DeviceSpec>().inputTimesliceId == 0) {
+      o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get<const o2::framework::DeviceSpec>().name, conf.getName()), conf.getName());
+      TMap md;
+      md.SetOwnerKeyValue();
+      md.Add(new TObjString(conf.getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(conf.getName()).c_str()));
+      pc.outputs().snapshot(Output{"META", "COSMICMATCHER", 0}, md);
+    }
+  }
   mTimer.Stop();
 }
 
@@ -193,6 +208,8 @@ DataProcessorSpec getCosmicsMatchingSpec(GTrackID::mask_t src, bool usePV, bool
   o2::tpc::VDriftHelper::requestCCDBInputs(dataRequest->inputs);
   dataRequest->inputs.emplace_back("corrMap", o2::header::gDataOriginTPC, "TPCCORRMAP", 0, Lifetime::Timeframe);
 
+  outputs.emplace_back("META", "COSMICMATCHER", 0, Lifetime::Sporadic);
+
   return DataProcessorSpec{
     "cosmics-matcher",
     dataRequest->inputs,

Detectors/GlobalTrackingWorkflow/src/PrimaryVertexingSpec.cxx

@@ -11,12 +11,15 @@
 
 /// @file  PrimaryVertexingSpec.cxx
 
+#include <TMap.h>
+#include <TObjString.h>
 #include <vector>
 #include <TStopwatch.h>
 #include "DataFormatsGlobalTracking/RecoContainer.h"
 #include "DataFormatsGlobalTracking/RecoContainerCreateTracksVariadic.h"
 #include "DataFormatsITSMFT/TrkClusRef.h"
 #include "DataFormatsCalibration/MeanVertexObject.h"
+#include "DataFormatsCalibration/MeanVertexBiasParam.h"
 #include "ReconstructionDataFormats/TrackTPCITS.h"
 #include "ReconstructionDataFormats/GlobalTrackID.h"
 #include "DetectorsBase/Propagator.h"
@@ -201,8 +204,16 @@ void PrimaryVertexingSpec::run(ProcessingContext& pc)
   static bool first = true;
   if (first) {
     first = false;
+    const auto& confPV = PVertexerParams::Instance();
+    const auto& confMV = o2::dataformats::MeanVertexBiasParam::Instance();
     if (pc.services().get<const o2::framework::DeviceSpec>().inputTimesliceId == 0) {
-      o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get<const o2::framework::DeviceSpec>().name, PVertexerParams::Instance().getName()), PVertexerParams::Instance().getName());
+      o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get<const o2::framework::DeviceSpec>().name, confPV.getName()), confPV.getName());
+      o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get<const o2::framework::DeviceSpec>().name, confMV.getName()), confMV.getName());
+      TMap md;
+      md.SetOwnerKeyValue();
+      md.Add(new TObjString(confPV.getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(confPV.getName()).c_str()));
+      md.Add(new TObjString(confMV.getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(confMV.getName()).c_str()));
+      pc.outputs().snapshot(Output{"META", "PVERTEXER", 0}, md);
     }
   }
 }
@@ -289,6 +300,8 @@ DataProcessorSpec getPrimaryVertexingSpec(GTrackID::mask_t src, bool skip, bool
                                                               true);
   dataRequest->inputs.emplace_back("meanvtx", "GLO", "MEANVERTEX", 0, Lifetime::Condition, ccdbParamSpec("GLO/Calib/MeanVertex", {}, 1));
 
+  outputs.emplace_back("META", "PVERTEXER", 0, Lifetime::Sporadic);
+
   return DataProcessorSpec{
     "primary-vertexing",
     dataRequest->inputs,

Detectors/GlobalTrackingWorkflow/src/SecondaryVertexingSpec.cxx

@@ -11,6 +11,8 @@
 
 /// @file  SecondaryVertexingSpec.cxx
 
+#include <TMap.h>
+#include <TObjString.h>
 #include <vector>
 #include "DataFormatsCalibration/MeanVertexObject.h"
 #include "Framework/CCDBParamSpec.h"
@@ -124,10 +126,17 @@ void SecondaryVertexingSpec::run(ProcessingContext& pc)
   if (first) {
     first = false;
     if (pc.services().get<const o2::framework::DeviceSpec>().inputTimesliceId == 0) {
-      o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get<const o2::framework::DeviceSpec>().name, SVertexerParams::Instance().getName()), SVertexerParams::Instance().getName());
+      const auto& confSV = SVertexerParams::Instance();
+      o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get<const o2::framework::DeviceSpec>().name, confSV.getName()), confSV.getName());
+      TMap md;
+      md.SetOwnerKeyValue();
+      md.Add(new TObjString(confSV.getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(confSV.getName()).c_str()));
       if (mEnableStrangenessTracking) {
-        o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get<const o2::framework::DeviceSpec>().name, o2::strangeness_tracking::StrangenessTrackingParamConfig::Instance().getName()), o2::strangeness_tracking::StrangenessTrackingParamConfig::Instance().getName());
+        const auto& confST = o2::strangeness_tracking::StrangenessTrackingParamConfig::Instance();
+        o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get<const o2::framework::DeviceSpec>().name, confST().getName()), confST.getName());
+        md.Add(new TObjString(confST.getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(confST.getName()).c_str()));
       }
+      pc.outputs().snapshot(Output{"META", "SVERTEXER", 0}, md);
     }
   }
 }
@@ -298,6 +307,7 @@ DataProcessorSpec getSecondaryVertexingSpec(GTrackID::mask_t src, bool enableCas
       LOG(info) << "Strangeness tracker will use MC";
     }
   }
+  outputs.emplace_back("META", "SVERTEXER", 0, Lifetime::Sporadic);
 
   return DataProcessorSpec{
     "secondary-vertexing",

Detectors/GlobalTrackingWorkflow/src/StrangenessTrackingSpec.cxx

@@ -11,6 +11,8 @@
 /// \file StrangenessTrackingSpec.cxx
 /// \brief
 
+#include <TMap.h>
+#include <TObjString.h>
 #include "TGeoGlobalMagField.h"
 #include "Framework/ConfigParamRegistry.h"
 #include "Field/MagneticField.h"
@@ -80,6 +82,18 @@ void StrangenessTrackerSpec::run(framework::ProcessingContext& pc)
     pc.outputs().snapshot(Output{"GLO", "STRANGETRACKS_MC", 0}, mTracker.getStrangeTrackLabels());
   }
 
+  static bool first = true;
+  if (first) {
+    first = false;
+    if (pc.services().get<const o2::framework::DeviceSpec>().inputTimesliceId == 0) {
+      const auto& conf = StrangenessTrackingParamConfig::Instance();
+      o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get<const o2::framework::DeviceSpec>().name, conf.getName()), conf.getName());
+      TMap md;
+      md.SetOwnerKeyValue();
+      md.Add(new TObjString(conf.getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(conf.getName()).c_str()));
+      pc.outputs().snapshot(Output{"META", "STRTRACKER", 0}, md);
+    }
+  }
   mTimer.Stop();
 }
 
@@ -162,6 +176,7 @@ DataProcessorSpec getStrangenessTrackerSpec(o2::dataformats::GlobalTrackID::mask
     outputs.emplace_back("GLO", "STRANGETRACKS_MC", 0, Lifetime::Timeframe);
     LOG(info) << "Strangeness tracker will use MC";
   }
+  outputs.emplace_back("META", "STRTRACKER", 0, Lifetime::Sporadic);
 
   return DataProcessorSpec{
     "strangeness-tracker",

Detectors/GlobalTrackingWorkflow/src/TOFMatcherSpec.cxx

@@ -13,6 +13,8 @@
 
 #include <vector>
 #include <string>
+#include <TMap.h>
+#include <TObjString.h>
 #include "TStopwatch.h"
 #include "Framework/ConfigParamRegistry.h"
 #include "DetectorsBase/GeometryManager.h"
@@ -218,7 +220,12 @@ void TOFMatcherSpec::run(ProcessingContext& pc)
   if (first) {
     first = false;
     if (pc.services().get<const o2::framework::DeviceSpec>().inputTimesliceId == 0) {
-      o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get<const o2::framework::DeviceSpec>().name, MatchTOFParams::Instance().getName()), MatchTOFParams::Instance().getName());
+      const auto& conf = MatchTOFParams::Instance();
+      o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get<const o2::framework::DeviceSpec>().name, conf.getName()), conf.getName());
+      TMap md;
+      md.SetOwnerKeyValue();
+      md.Add(new TObjString(conf.getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(conf.getName()).c_str()));
+      pc.outputs().snapshot(Output{"META", "TOFMATCHER", 0}, md);
     }
   }
 
@@ -309,6 +316,7 @@ DataProcessorSpec getTOFMatcherSpec(GID::mask_t src, bool useMC, bool useFIT, bo
     outputs.emplace_back(o2::header::gDataOriginTOF, "MATCHABLES_16", 0, Lifetime::Timeframe);
     outputs.emplace_back(o2::header::gDataOriginTOF, "MATCHABLES_17", 0, Lifetime::Timeframe);
   }
+  outputs.emplace_back("META", "TOFMATCHER", 0, Lifetime::Sporadic);
 
   return DataProcessorSpec{
     "tof-matcher",