From 46a6b0c5e97eca7389b6618e9dd0688154aba1f7 Mon Sep 17 00:00:00 2001 From: Luke Pickering Date: Fri, 12 Dec 2025 13:17:02 +0000 Subject: [PATCH 01/14] working on it --- Parameters/CMakeLists.txt | 3 +- Parameters/PCAHandler.cpp | 664 ++++++----- Parameters/PCAHandler.h | 20 +- Parameters/ParameterHandlerBase.cpp | 1536 +++++++++++-------------- Parameters/ParameterHandlerBase.h | 513 +++++---- Parameters/ParameterHandlerUtils.h | 1132 ++++++++++-------- Parameters/ParameterTunes.h | 5 +- cmake/Modules/MaCh3Dependencies.cmake | 8 + 8 files changed, 1996 insertions(+), 1885 deletions(-) diff --git a/Parameters/CMakeLists.txt b/Parameters/CMakeLists.txt index e61a9b7f7..a5bfc2928 100644 --- a/Parameters/CMakeLists.txt +++ b/Parameters/CMakeLists.txt @@ -27,7 +27,7 @@ if(MaCh3_GPU_ENABLED) set_property(TARGET Parameters PROPERTY CUDA_ARCHITECTURES ${CMAKE_CUDA_ARCHITECTURES}) endif() -target_link_libraries(Parameters PUBLIC Manager) +target_link_libraries(Parameters PUBLIC Manager Eigen3::Eigen) # KS: ROOT ≤ 6.30 requires explicit linking of MathMore (if built, not sure why through...) if(ROOT_VERSION VERSION_LESS 6.30) if(TARGET ROOT::MathMore) @@ -36,6 +36,7 @@ if(ROOT_VERSION VERSION_LESS 6.30) message(WARNING "ROOT::MathMore target not found; continuing without it") endif() endif() + target_link_libraries(Parameters PRIVATE MaCh3Warnings) target_include_directories(Parameters PUBLIC diff --git a/Parameters/PCAHandler.cpp b/Parameters/PCAHandler.cpp index 0b945efe9..1e2d69d6e 100755 --- a/Parameters/PCAHandler.cpp +++ b/Parameters/PCAHandler.cpp @@ -1,60 +1,44 @@ #include "Parameters/PCAHandler.h" - -// ******************************************** -PCAHandler::PCAHandler() { -// ******************************************** - _pCurrVal = nullptr; - _pPropVal = nullptr; -} - -// ******************************************** -PCAHandler::~PCAHandler() { -// ******************************************** -} - -// ******************************************** -void PCAHandler::SetupPointers(std::vector* fCurr_Val, - std::vector* fProp_Val) { -// ******************************************** - _pCurrVal = fCurr_Val; - _pPropVal = fProp_Val; -} - -// ******************************************** -void PCAHandler::ConstructPCA(TMatrixDSym* CovMatrix, const int firstPCAd, const int lastPCAd, - const double eigen_thresh, const int _fNumPar) { // ******************************************** +void PCAHandler::ConstructPCA(Eigen::MatrixXd const &CovMatrix, + const int firstPCAd, const int lastPCAd, + const double eigen_thresh) { + // ******************************************** FirstPCAdpar = firstPCAd; LastPCAdpar = lastPCAd; eigen_threshold = eigen_thresh; // Check that covariance matrix exists - if (CovMatrix == nullptr) { + if (CovMatrix == NULL) { MACH3LOG_ERROR("Covariance matrix for has not yet been set"); MACH3LOG_ERROR("Can not construct PCA until it is set"); - throw MaCh3Exception(__FILE__ , __LINE__ ); + throw MaCh3Exception(__FILE__, __LINE__); } - if(FirstPCAdpar > CovMatrix->GetNrows()-1 || LastPCAdpar>CovMatrix->GetNrows()-1) { - MACH3LOG_ERROR("FirstPCAdpar and LastPCAdpar are higher than the number of parameters"); - MACH3LOG_ERROR("first: {} last: {}, params: {}", FirstPCAdpar, LastPCAdpar, CovMatrix->GetNrows()-1); - throw MaCh3Exception(__FILE__ , __LINE__ ); + if ((FirstPCAdpar >= CovMatrix.rows()) || (LastPCAdpar >= CovMatrix.rows())) { + MACH3LOG_ERROR("FirstPCAdpar and LastPCAdpar are higher than the number of " + "parameters"); + MACH3LOG_ERROR("first: {} last: {}, params: {}", FirstPCAdpar, LastPCAdpar, + CovMatrix.rows()); + throw MaCh3Exception(__FILE__, __LINE__); } - if(FirstPCAdpar < 0 || LastPCAdpar < 0){ - MACH3LOG_ERROR("FirstPCAdpar and LastPCAdpar are less than 0 but not default -999"); + if ((FirstPCAdpar < 0) || (LastPCAdpar < 0)) { + MACH3LOG_ERROR( + "FirstPCAdpar and LastPCAdpar are less than 0 but not default -999"); MACH3LOG_ERROR("first: {} last: {}", FirstPCAdpar, LastPCAdpar); - throw MaCh3Exception(__FILE__ , __LINE__ ); + throw MaCh3Exception(__FILE__, __LINE__); } - MACH3LOG_INFO("PCAing parameters {} through {} inclusive", FirstPCAdpar, LastPCAdpar); - int NumUnPCAdPars = CovMatrix->GetNrows()-(LastPCAdpar-FirstPCAdpar+1); + MACH3LOG_INFO("PCAing parameters {} through {} inclusive", FirstPCAdpar, + LastPCAdpar); + int numunpcadpars = CovMatrix->GetNrows() - (LastPCAdpar - FirstPCAdpar + 1); - // KS: Make sure we are not doing anything silly with PCA - SanitisePCA(CovMatrix); + M3::EnsureNoOutOfBlockCorrelations(CovMatrix, FirstPCAdpar, LastPCAdpar); - TMatrixDSym submat(CovMatrix->GetSub(FirstPCAdpar,LastPCAdpar,FirstPCAdpar,LastPCAdpar)); + TMatrixDSym submat( + CovMatrix->GetSub(FirstPCAdpar, LastPCAdpar, FirstPCAdpar, LastPCAdpar)); - //CW: Calculate how many eigen values this threshold corresponds to + // CW: Calculate how many eigen values this threshold corresponds to TMatrixDSymEigen eigen(submat); eigen_values.ResizeTo(eigen.GetEigenValues()); eigen_vectors.ResizeTo(eigen.GetEigenVectors()); @@ -66,51 +50,66 @@ void PCAHandler::ConstructPCA(TMatrixDSym* CovMatrix, const int firstPCAd, const sum += eigen_values(i); } nKeptPCApars = eigen_values.GetNrows(); - //CW: Now go through again and see how many eigen values correspond to threshold + // CW: Now go through again and see how many eigen values correspond to + // threshold for (int i = 0; i < eigen_values.GetNrows(); ++i) { // Get the relative size of the eigen value - double sig = eigen_values(i)/sum; + double sig = eigen_values(i) / sum; // Check against the threshold if (sig < eigen_threshold) { nKeptPCApars = i; break; } } - NumParPCA = NumUnPCAdPars+nKeptPCApars; - MACH3LOG_INFO("Threshold of {} on eigen values relative sum of eigen value ({}) generates {} eigen vectors, plus we have {} unpcad pars, for a total of {}", eigen_threshold, sum, nKeptPCApars, NumUnPCAdPars, NumParPCA); - //DB Create array of correct size so eigen_values can be used in CorrelateSteps + NumParPCA = numunpcadpars + nKeptPCApars; + MACH3LOG_INFO("Threshold of {} on eigen values relative sum of eigen value " + "({}) generates {} eigen vectors, plus we have {} unpcad pars, " + "for a total of {}", + eigen_threshold, sum, nKeptPCApars, numunpcadpars, NumParPCA); + + // DB Create array of correct size so eigen_values can be used in + // CorrelateSteps eigen_values_master = std::vector(NumParPCA, 1.0); - for (int i = FirstPCAdpar; i < FirstPCAdpar+nKeptPCApars; ++i) {eigen_values_master[i] = eigen_values(i-FirstPCAdpar);} + for (int i = FirstPCAdpar; i < FirstPCAdpar + nKeptPCApars; ++i) { + eigen_values_master[i] = eigen_values(i - FirstPCAdpar); + } // Now construct the transfer matrices - //These matrices will be as big as number of unPCAd pars plus number of eigenvalues kept + // These matrices will be as big as number of unPCAd pars plus number of + // eigenvalues kept TransferMat.ResizeTo(CovMatrix->GetNrows(), NumParPCA); TransferMatT.ResizeTo(CovMatrix->GetNrows(), NumParPCA); // Get a subset of the eigen vector matrix - TMatrixD temp(eigen_vectors.GetSub(0, eigen_vectors.GetNrows()-1, 0, nKeptPCApars-1)); + TMatrixD temp(eigen_vectors.GetSub(0, eigen_vectors.GetNrows() - 1, 0, + nKeptPCApars - 1)); - //Make transfer matrix which is two blocks of identity with a block of the PCA transfer matrix in between + // Make transfer matrix which is two blocks of identity with a block of the + // PCA transfer matrix in between TMatrixD temp2; temp2.ResizeTo(CovMatrix->GetNrows(), NumParPCA); - //First set the whole thing to 0 - temp2.Zero(); - - //Set the first identity block for non-PCAed params before PCA block, before PCA XRows == YRows - for(int iRow = 0; iRow < FirstPCAdpar; iRow++) { - temp2(iRow, iRow) = 1.0; + // First set the whole thing to 0 + for (int iRow = 0; iRow < CovMatrix->GetNrows(); iRow++) { + for (int iCol = 0; iCol < NumParPCA; iCol++) { + temp2[iRow][iCol] = 0; + } + } + // Set the first identity block + if (FirstPCAdpar != 0) { + for (int iRow = 0; iRow < FirstPCAdpar; iRow++) { + temp2[iRow][iRow] = 1; + } } - //Set the transfer matrix block for the PCAd pars - temp2.SetSub(FirstPCAdpar,FirstPCAdpar,temp); + // Set the transfer matrix block for the PCAd pars + temp2.SetSub(FirstPCAdpar, FirstPCAdpar, temp); - //Set the second identity block starting after PCA block, remember XRows != YRows. - // XRows -> normal base, YRows, PCA base - for(int iRow = 0;iRow < (CovMatrix->GetNrows()-1)-LastPCAdpar; iRow++) { - const int OrigRow = LastPCAdpar + 1 + iRow; - const int PCARow = FirstPCAdpar + nKeptPCApars + iRow; - temp2(OrigRow, PCARow) = 1.; + // Set the second identity block + if (LastPCAdpar != CovMatrix.rows() - 1) { + for (int iRow = 0; iRow < (CovMatrix.rows() - 1) - LastPCAdpar; iRow++) { + temp2[LastPCAdpar + 1 + iRow][FirstPCAdpar + nKeptPCApars + iRow] = 1; + } } TransferMat = temp2; @@ -119,69 +118,38 @@ void PCAHandler::ConstructPCA(TMatrixDSym* CovMatrix, const int firstPCAd, const // And then transpose TransferMatT.T(); +#ifdef DEBUG_PCA + // KS: Let's dump all useful matrices to properly validate PCA + DebugPCA(sum, temp, submat, CovMatrix->GetNrows()); +#endif + // Make the PCA parameter arrays _fParCurrPCA.ResizeTo(NumParPCA); _fParPropPCA.ResizeTo(NumParPCA); _fPreFitValuePCA.resize(NumParPCA); - //KS: make easy map so we could easily find un-decomposed parameters - _fErrorPCA.assign(NumParPCA, 1); - isDecomposedPCA.assign(NumParPCA, -1); - // First non PCA-ed block, since this is before PCA-ed block we don't need any mapping - for (int i = 0; i < FirstPCAdpar; ++i) { - isDecomposedPCA[i] = i; - } - - // Second non-PCA-ed block, keep in mind this is in PCA base so we cannot use LastPCAdpar - // we must shift them back into the original parameter index range. - const int shift = _fNumPar - NumParPCA; - for (int i = FirstPCAdpar + nKeptPCApars; i < NumParPCA; ++i) { - isDecomposedPCA[i] = i + shift; - } - - #ifdef DEBUG_PCA - //KS: Let's dump all useful matrices to properly validate PCA - DebugPCA(sum, temp, submat, CovMatrix->GetNrows()); - #endif -} - -// ******************************************** -// Make sure decomposed matrix isn't correlated with undecomposed -void PCAHandler::SanitisePCA(TMatrixDSym* CovMatrix) { -// ******************************************** - constexpr double correlation_threshold = 1e-6; - - bool found_significant_correlation = false; - - int N = CovMatrix->GetNrows(); - for (int i = FirstPCAdpar; i <= LastPCAdpar; ++i) { - for (int j = 0; j < N; ++j) { - // Skip if j is inside the decomposed range (we only want cross-correlations) - if (j >= FirstPCAdpar && j <= LastPCAdpar) continue; - - double corr_val = (*CovMatrix)(i, j); - if (std::fabs(corr_val) > correlation_threshold) { - found_significant_correlation = true; - MACH3LOG_ERROR("Significant correlation detected between decomposed parameter '{}' " - "and undecomposed parameter '{}': {:.6e}", i, j, corr_val); - } - } + // KS: make easy map so we could easily find un-decomposed parameters + isDecomposedPCA.resize(NumParPCA); + _fErrorPCA.resize(NumParPCA); + for (int i = 0; i < NumParPCA; ++i) { + _fErrorPCA[i] = 1; + isDecomposedPCA[i] = -1; } + for (int i = 0; i < FirstPCAdpar; ++i) + isDecomposedPCA[i] = i; - if (found_significant_correlation) { - MACH3LOG_ERROR("There are correlations between undecomposed and decomposed part of matrices, this will not work"); - throw MaCh3Exception(__FILE__ , __LINE__); - } + for (int i = FirstPCAdpar + nKeptPCApars + 1; i < NumParPCA; ++i) + isDecomposedPCA[i] = i + (_fNumPar - NumParPCA); } // ******************************************** // Update so that current step becomes the previously proposed step void PCAHandler::AcceptStep() _noexcept_ { -// ******************************************** + // ******************************************** // Update the book-keeping for the output - #ifdef MULTITHREAD - #pragma omp parallel for - #endif +#ifdef MULTITHREAD +#pragma omp parallel for +#endif for (int i = 0; i < NumParPCA; ++i) { _fParCurrPCA(i) = _fParPropPCA(i); } @@ -190,35 +158,34 @@ void PCAHandler::AcceptStep() _noexcept_ { } // ************************************************ -// Correlate the steps by setting the proposed step of a parameter to its current value + some correlated throw -void PCAHandler::CorrelateSteps(const std::vector& IndivStepScale, - const double GlobalStepScale, - const double* _restrict_ randParams, - const double* _restrict_ corr_throw) _noexcept_ { -// ************************************************ +// Correlate the steps by setting the proposed step of a parameter to its +// current value + some correlated throw +void PCAHandler::CorrelateSteps( + const std::vector &IndivStepScale, const double GlobalStepScale, + const double *_restrict_ randParams, + const double *_restrict_ corr_throw) _noexcept_ { + // ************************************************ // Throw around the current step - #ifdef MULTITHREAD - #pragma omp parallel for - #endif - for (int i = 0; i < NumParPCA; ++i) - { - if (_fErrorPCA[i] > 0.) - { +#ifdef MULTITHREAD +#pragma omp parallel for +#endif + for (int i = 0; i < NumParPCA; ++i) { + if (_fErrorPCA[i] > 0.) { double IndStepScale = 1.; - //KS: If undecomposed parameter apply individual step scale and Cholesky for better acceptance rate - if(isDecomposedPCA[i] >= 0) - { + // KS: If undecomposed parameter apply individual step scale and Cholesky + // for better acceptance rate + if (isDecomposedPCA[i] >= 0) { IndStepScale *= IndivStepScale[isDecomposedPCA[i]]; IndStepScale *= corr_throw[isDecomposedPCA[i]]; } - //If decomposed apply only random number - else - { + // If decomposed apply only random number + else { IndStepScale *= randParams[i]; - //KS: All PCA-ed parameters have the same step scale + // KS: All PCA-ed parameters have the same step scale IndStepScale *= IndivStepScale[FirstPCAdpar]; } - _fParPropPCA(i) = _fParCurrPCA(i)+GlobalStepScale*IndStepScale*eigen_values_master[i]; + _fParPropPCA(i) = _fParCurrPCA(i) + + GlobalStepScale * IndStepScale * eigen_values_master[i]; } } // Then update the parameter basis @@ -228,44 +195,47 @@ void PCAHandler::CorrelateSteps(const std::vector& IndivStepScale, // ******************************************** // Transfer a parameter variation in the parameter basis to the eigen basis void PCAHandler::TransferToPCA() { -// ******************************************** + // ******************************************** // Make the temporary vectors TVectorD fParCurr_vec(static_cast(_pCurrVal->size())); TVectorD fParProp_vec(static_cast(_pCurrVal->size())); - for(int i = 0; i < static_cast(_pCurrVal->size()); ++i) { + for (int i = 0; i < static_cast(_pCurrVal->size()); ++i) { fParCurr_vec(i) = (*_pCurrVal)[i]; fParProp_vec(i) = (*_pPropVal)[i]; } - _fParCurrPCA = TransferMatT*fParCurr_vec; - _fParPropPCA = TransferMatT*fParProp_vec; + _fParCurrPCA = TransferMatT * fParCurr_vec; + _fParPropPCA = TransferMatT * fParProp_vec; } // ******************************************** -void PCAHandler::SetInitialParameters(std::vector& IndStepScale) { -// ******************************************** +void PCAHandler::SetInitialParameters(std::vector &IndStepScale) { + // ******************************************** TransferToPCA(); for (int i = 0; i < NumParPCA; ++i) { _fPreFitValuePCA[i] = _fParCurrPCA(i); } - //DB Set Individual Step scale for PCA parameters to the LastPCAdpar fIndivStepScale because the step scale for those parameters is set by 'eigen_values[i]' but needs an overall step scale - // However, individual step scale for non-PCA parameters needs to be set correctly + // DB Set Individual Step scale for PCA parameters to the LastPCAdpar + // fIndivStepScale because the step scale for those parameters is set by + // 'eigen_values[i]' but needs an overall step scale + // However, individual step scale for non-PCA parameters needs to be set + // correctly for (int i = FirstPCAdpar; i <= LastPCAdpar; i++) { - IndStepScale[i] = IndStepScale[LastPCAdpar-1]; + IndStepScale[i] = IndStepScale[LastPCAdpar - 1]; } } // ******************************************** // Transfer a parameter variation in the eigen basis to the parameter basis void PCAHandler::TransferToParam() { -// ******************************************** + // ******************************************** // Make the temporary vectors - TVectorD fParProp_vec = TransferMat*_fParPropPCA; - TVectorD fParCurr_vec = TransferMat*_fParCurrPCA; - #ifdef MULTITHREAD - #pragma omp parallel for - #endif - for(int i = 0; i < static_cast(_pCurrVal->size()); ++i) { + TVectorD fParProp_vec = TransferMat * _fParPropPCA; + TVectorD fParCurr_vec = TransferMat * _fParCurrPCA; +#ifdef MULTITHREAD +#pragma omp parallel for +#endif + for (int i = 0; i < static_cast(_pCurrVal->size()); ++i) { (*_pPropVal)[i] = fParProp_vec(i); (*_pCurrVal)[i] = fParCurr_vec(i); } @@ -273,11 +243,12 @@ void PCAHandler::TransferToParam() { // ******************************************** // Throw the proposed parameter by mag sigma. -void PCAHandler::ThrowParProp(const double mag, const double* _restrict_ randParams) { -// ******************************************** +void PCAHandler::ThrowParProp(const double mag, + const double *_restrict_ randParams) { + // ******************************************** for (int i = 0; i < NumParPCA; i++) { if (_fErrorPCA[i] > 0.) { - _fParPropPCA(i) = _fParCurrPCA(i)+mag*randParams[i]; + _fParPropPCA(i) = _fParCurrPCA(i) + mag * randParams[i]; } } TransferToParam(); @@ -285,96 +256,112 @@ void PCAHandler::ThrowParProp(const double mag, const double* _restrict_ randPar // ******************************************** // Throw the proposed parameter by mag sigma. -void PCAHandler::ThrowParCurr(const double mag, const double* _restrict_ randParams) { -// ******************************************** +void PCAHandler::ThrowParCurr(const double mag, + const double *_restrict_ randParams) { + // ******************************************** for (int i = 0; i < NumParPCA; i++) { if (_fErrorPCA[i] > 0.) { - _fParPropPCA(i) = mag*randParams[i]; + _fParPropPCA(i) = mag * randParams[i]; } } TransferToParam(); } // ******************************************** -void PCAHandler::Print() const { -// ******************************************** +void PCAHandler::Print() { + // ******************************************** MACH3LOG_INFO("PCA:"); for (int i = 0; i < NumParPCA; ++i) { - MACH3LOG_INFO("PCA {:<2} Current: {:<10.2f} Proposed: {:<10.2f}", i, _fParCurrPCA(i), _fParPropPCA(i)); + MACH3LOG_INFO("PCA {:<2} Current: {:<10.2f} Proposed: {:<10.2f}", i, + _fParCurrPCA(i), _fParPropPCA(i)); } } // ******************************************** -void PCAHandler::SetBranches(TTree &tree, bool SaveProposal, const std::vector& Names) { -// ******************************************** +void PCAHandler::SetBranches(TTree &tree, bool SaveProposal, + const std::vector &Names) { + // ******************************************** for (int i = 0; i < NumParPCA; ++i) { - tree.Branch(Form("%s_PCA", Names[i].c_str()), &_fParCurrPCA.GetMatrixArray()[i], Form("%s_PCA/D", Names[i].c_str())); + tree.Branch(Form("%s_PCA", Names[i].c_str()), + &_fParCurrPCA.GetMatrixArray()[i], + Form("%s_PCA/D", Names[i].c_str())); } - if(SaveProposal) - { + if (SaveProposal) { for (int i = 0; i < NumParPCA; ++i) { - tree.Branch(Form("%s_PCA_Prop", Names[i].c_str()), &_fParPropPCA.GetMatrixArray()[i], Form("%s_PCA_Prop/D", Names[i].c_str())); + tree.Branch(Form("%s_PCA_Prop", Names[i].c_str()), + &_fParPropPCA.GetMatrixArray()[i], + Form("%s_PCA_Prop/D", Names[i].c_str())); } } } // ******************************************** -void PCAHandler::ToggleFixAllParameters(const std::vector& Names) { -// ******************************************** - for (int i = 0; i < NumParPCA; i++) ToggleFixParameter(i, Names); +void PCAHandler::ToggleFixAllParameters(const std::vector &Names) { + // ******************************************** + for (int i = 0; i < NumParPCA; i++) + ToggleFixParameter(i, Names); } // ******************************************** -void PCAHandler::ToggleFixParameter(const int i, const std::vector& Names) { -// ******************************************** +void PCAHandler::ToggleFixParameter(const int i, + const std::vector &Names) { + // ******************************************** int isDecom = -1; for (int im = 0; im < NumParPCA; ++im) { - if(isDecomposedPCA[im] == i) {isDecom = im;} + if (isDecomposedPCA[im] == i) { + isDecom = im; + } } - if(isDecom < 0) { + if (isDecom < 0) { MACH3LOG_ERROR("Parameter {} is PCA decomposed can't fix this", Names[i]); - //throw MaCh3Exception(__FILE__ , __LINE__ ); + // throw MaCh3Exception(__FILE__ , __LINE__ ); } else { _fErrorPCA[isDecom] *= -1.0; - if(IsParameterFixedPCA(i)) MACH3LOG_INFO("Setting un-decomposed {}(parameter {}/{} in PCA base) to fixed at {}", Names[i], i, isDecom, (*_pCurrVal)[i]); - else MACH3LOG_INFO("Setting un-decomposed {}(parameter {}/{} in PCA base) free", Names[i], i, isDecom); + if (IsParameterFixedPCA(i)) + MACH3LOG_INFO("Setting un-decomposed {}(parameter {}/{} in PCA base) to " + "fixed at {}", + Names[i], i, isDecom, (*_pCurrVal)[i]); + else + MACH3LOG_INFO( + "Setting un-decomposed {}(parameter {}/{} in PCA base) free", + Names[i], i, isDecom); } } - -// ******************************************** -void PCAHandler::ThrowParameters(const std::vector>& random_number, - double** throwMatrixCholDecomp, - double* randParams, - double* corr_throw, - const std::vector& fPreFitValue, - const std::vector& fLowBound, - const std::vector& fUpBound, - const int _fNumPar) { // ******************************************** - //KS: Do not multithread! +void PCAHandler::ThrowParameters( + const std::vector> &random_number, + double **throwMatrixCholDecomp, double *randParams, double *corr_throw, + const std::vector &fPreFitValue, + const std::vector &fLowBound, const std::vector &fUpBound, + const int _fNumPar) { + // ******************************************** + // KS: Do not multithread! for (int i = 0; i < NumParPCA; ++i) { // Check if parameter is fixed first: if so don't randomly throw - if (IsParameterFixedPCA(i)) continue; + if (IsParameterFixedPCA(i)) + continue; - if(!IsParameterDecomposed(i)) - { + if (!IsParameterDecomposed(i)) { (*_pPropVal)[i] = fPreFitValue[i] + corr_throw[i]; int throws = 0; - // Try again if we the initial parameter proposal falls outside of the range of the parameter + // Try again if we the initial parameter proposal falls outside of the + // range of the parameter while ((*_pPropVal)[i] > fUpBound[i] || (*_pPropVal)[i] < fLowBound[i]) { randParams[i] = random_number[M3::GetThreadIndex()]->Gaus(0, 1); - const double corr_throw_single = M3::MatrixVectorMultiSingle(throwMatrixCholDecomp, randParams, _fNumPar, i); + const double corr_throw_single = M3::MatrixVectorMultiSingle( + throwMatrixCholDecomp, randParams, _fNumPar, i); (*_pPropVal)[i] = fPreFitValue[i] + corr_throw_single; - if (throws > 10000) - { - //KS: Since we are multithreading there is danger that those messages - //will be all over the place, small price to pay for faster code - MACH3LOG_WARN("Tried {} times to throw parameter {} but failed", throws, i); - MACH3LOG_WARN("Setting _fPropVal: {} to {}", (*_pPropVal)[i], fPreFitValue[i]); + if (throws > 10000) { + // KS: Since we are multithreading there is danger that those messages + // will be all over the place, small price to pay for faster code + MACH3LOG_WARN("Tried {} times to throw parameter {} but failed", + throws, i); + MACH3LOG_WARN("Setting _fPropVal: {} to {}", (*_pPropVal)[i], + fPreFitValue[i]); MACH3LOG_WARN("I live at {}:{}", __FILE__, __LINE__); - (*_pPropVal)[i] = fPreFitValue[i]; + (*_pPropVal)[i] = fPreFitValue[i]; } throws++; } @@ -382,14 +369,18 @@ void PCAHandler::ThrowParameters(const std::vector>& r } else { // KS: We have to multiply by number of parameters in PCA base - SetParPropPCA(i, GetPreFitValuePCA(i) + randParams[i] * eigen_values_master[i] * (LastPCAdpar - FirstPCAdpar)); + SetParPropPCA(i, GetPreFitValuePCA(i) + randParams[i] * + eigen_values_master[i] * + (LastPCAdpar - FirstPCAdpar)); SetParCurrPCA(i, GetParPropPCA(i)); } } // end of parameter loop - /// @todo KS: We don't check if param is out of bounds. This is more problematic for PCA params. + /// @todo KS: We don't check if param is out of bounds. This is more + /// problematic for PCA params. for (int i = 0; i < _fNumPar; ++i) { - (*_pPropVal)[i] = std::max(fLowBound[i], std::min((*_pPropVal)[i], fUpBound[i])); + (*_pPropVal)[i] = + std::max(fLowBound[i], std::min((*_pPropVal)[i], fUpBound[i])); (*_pCurrVal)[i] = (*_pPropVal)[i]; } } @@ -397,62 +388,61 @@ void PCAHandler::ThrowParameters(const std::vector>& r #ifdef DEBUG_PCA #pragma GCC diagnostic ignored "-Wfloat-conversion" // ******************************************** -//KS: Let's dump all useful matrices to properly validate PCA -void PCAHandler::DebugPCA(const double sum, TMatrixD temp, TMatrixDSym submat, int NumPar) { -// ******************************************** - int originalErrorWarning = gErrorIgnoreLevel; - gErrorIgnoreLevel = kFatal; - - (void)submat;//This is used if DEBUG_PCA==2, this hack is to avoid compiler warnings - for (int i = 0; i < NumParPCA; ++i) { - MACH3LOG_DEBUG("Param {} isDecomposedPCA={}", i, isDecomposedPCA[i]); - } - +// KS: Let's dump all useful matrices to properly validate PCA +void PCAHandler::DebugPCA(const double sum, TMatrixD temp, TMatrixDSym submat, + int NumPar) { + // ******************************************** + (void)submat; // This is used if DEBUG_PCA==2, this hack is to avoid compiler + // warnings TFile *PCA_Debug = new TFile("Debug_PCA.root", "RECREATE"); PCA_Debug->cd(); bool PlotText = true; - //KS: If we have more than 200 plot becomes unreadable :( - if(NumPar > 200) PlotText = false; - - auto heigen_values = std::make_unique("eigen_values", "Eigen Values", eigen_values.GetNrows(), 0.0, eigen_values.GetNrows()); - heigen_values->SetDirectory(nullptr); - auto heigen_cumulative = std::make_unique("heigen_cumulative", "heigen_cumulative", eigen_values.GetNrows(), 0.0, eigen_values.GetNrows()); - heigen_cumulative->SetDirectory(nullptr); - auto heigen_frac = std::make_unique("heigen_fractional", "heigen_fractional", eigen_values.GetNrows(), 0.0, eigen_values.GetNrows()); - heigen_frac->SetDirectory(nullptr); + // KS: If we have more than 200 plot becomes unreadable :( + if (NumPar > 200) + PlotText = false; + + TH1D *heigen_values = + new TH1D("eigen_values", "Eigen Values", eigen_values.GetNrows(), 0.0, + eigen_values.GetNrows()); + TH1D *heigen_cumulative = + new TH1D("heigen_cumulative", "heigen_cumulative", + eigen_values.GetNrows(), 0.0, eigen_values.GetNrows()); + TH1D *heigen_frac = + new TH1D("heigen_fractional", "heigen_fractional", + eigen_values.GetNrows(), 0.0, eigen_values.GetNrows()); heigen_values->GetXaxis()->SetTitle("Eigen Vector"); heigen_values->GetYaxis()->SetTitle("Eigen Value"); double Cumulative = 0; - for(int i = 0; i < eigen_values.GetNrows(); i++) - { - heigen_values->SetBinContent(i+1, (eigen_values)(i)); - heigen_cumulative->SetBinContent(i+1, (eigen_values)(i)/sum + Cumulative); - heigen_frac->SetBinContent(i+1, (eigen_values)(i)/sum); - Cumulative += (eigen_values)(i)/sum; + for (int i = 0; i < eigen_values.GetNrows(); i++) { + heigen_values->SetBinContent(i + 1, (eigen_values)(i)); + heigen_cumulative->SetBinContent(i + 1, + (eigen_values)(i) / sum + Cumulative); + heigen_frac->SetBinContent(i + 1, (eigen_values)(i) / sum); + Cumulative += (eigen_values)(i) / sum; } heigen_values->Write("heigen_values"); eigen_values.Write("eigen_values"); heigen_cumulative->Write("heigen_values_cumulative"); heigen_frac->Write("heigen_values_frac"); - TH2D* heigen_vectors = new TH2D(eigen_vectors); + TH2D *heigen_vectors = new TH2D(eigen_vectors); heigen_vectors->GetXaxis()->SetTitle("Parameter in Normal Base"); heigen_vectors->GetYaxis()->SetTitle("Parameter in Decomposed Base"); heigen_vectors->Write("heigen_vectors"); eigen_vectors.Write("eigen_vectors"); - TH2D* SubsetPCA = new TH2D(temp); + TH2D *SubsetPCA = new TH2D(temp); SubsetPCA->GetXaxis()->SetTitle("Parameter in Normal Base"); SubsetPCA->GetYaxis()->SetTitle("Parameter in Decomposed Base"); SubsetPCA->Write("hSubsetPCA"); temp.Write("SubsetPCA"); - TH2D* hTransferMat = new TH2D(TransferMat); + TH2D *hTransferMat = new TH2D(TransferMat); hTransferMat->GetXaxis()->SetTitle("Parameter in Normal Base"); hTransferMat->GetYaxis()->SetTitle("Parameter in Decomposed Base"); - TH2D* hTransferMatT = new TH2D(TransferMatT); + TH2D *hTransferMatT = new TH2D(TransferMatT); hTransferMatT->GetXaxis()->SetTitle("Parameter in Decomposed Base"); hTransferMatT->GetYaxis()->SetTitle("Parameter in Normal Base"); @@ -462,7 +452,7 @@ void PCAHandler::DebugPCA(const double sum, TMatrixD temp, TMatrixDSym submat, i hTransferMatT->Write("hTransferMatT"); TransferMatT.Write("TransferMatT"); - auto c1 = std::make_unique("c1", " ", 0, 0, 1024, 1024); + TCanvas *c1 = new TCanvas("c1", " ", 0, 0, 1024, 1024); c1->SetBottomMargin(0.1); c1->SetTopMargin(0.05); c1->SetRightMargin(0.05); @@ -473,12 +463,12 @@ void PCAHandler::DebugPCA(const double sum, TMatrixD temp, TMatrixDSym submat, i gStyle->SetOptFit(0); gStyle->SetOptStat(0); // Make pretty correlation colors (red to blue) - constexpr int NRGBs = 5; + const int NRGBs = 5; TColor::InitializeColors(); - Double_t stops[NRGBs] = { 0.00, 0.25, 0.50, 0.75, 1.00 }; - Double_t red[NRGBs] = { 0.00, 0.25, 1.00, 1.00, 0.50 }; - Double_t green[NRGBs] = { 0.00, 0.25, 1.00, 0.25, 0.00 }; - Double_t blue[NRGBs] = { 0.50, 1.00, 1.00, 0.25, 0.00 }; + Double_t stops[NRGBs] = {0.00, 0.25, 0.50, 0.75, 1.00}; + Double_t red[NRGBs] = {0.00, 0.25, 1.00, 1.00, 0.50}; + Double_t green[NRGBs] = {0.00, 0.25, 1.00, 0.25, 0.00}; + Double_t blue[NRGBs] = {0.50, 1.00, 1.00, 0.25, 0.00}; TColor::CreateGradientColorTable(5, stops, red, green, blue, 255); gStyle->SetNumberContours(255); @@ -486,14 +476,15 @@ void PCAHandler::DebugPCA(const double sum, TMatrixD temp, TMatrixDSym submat, i double minz = 0; c1->Print("Debug_PCA.pdf["); - auto EigenLine = std::make_unique(nKeptPCApars, 0, nKeptPCApars, heigen_cumulative->GetMaximum()); + TLine *EigenLine = + new TLine(nKeptPCApars, 0, nKeptPCApars, heigen_cumulative->GetMaximum()); EigenLine->SetLineColor(kPink); EigenLine->SetLineWidth(2); EigenLine->SetLineStyle(kSolid); - auto text = std::make_unique(0.5, 0.5, Form("Threshold = %g", eigen_threshold)); - text->SetTextFont (43); - text->SetTextSize (40); + TText *text = new TText(0.5, 0.5, Form("Threshold = %g", eigen_threshold)); + text->SetTextFont(43); + text->SetTextSize(40); heigen_values->SetLineColor(kRed); heigen_values->SetLineWidth(2); @@ -503,18 +494,19 @@ void PCAHandler::DebugPCA(const double sum, TMatrixD temp, TMatrixDSym submat, i heigen_frac->SetLineWidth(2); c1->SetLogy(); - heigen_values->SetMaximum(heigen_cumulative->GetMaximum()+heigen_cumulative->GetMaximum()*0.4); + heigen_values->SetMaximum(heigen_cumulative->GetMaximum() + + heigen_cumulative->GetMaximum() * 0.4); heigen_values->Draw(); heigen_frac->Draw("SAME"); heigen_cumulative->Draw("SAME"); EigenLine->Draw("Same"); - text->DrawTextNDC(0.42, 0.84,Form("Threshold = %g", eigen_threshold)); + text->DrawTextNDC(0.42, 0.84, Form("Threshold = %g", eigen_threshold)); - auto leg = std::make_unique(0.2, 0.2, 0.6, 0.5); + TLegend *leg = new TLegend(0.2, 0.2, 0.6, 0.5); leg->SetTextSize(0.04); - leg->AddEntry(heigen_values.get(), "Absolute", "l"); - leg->AddEntry(heigen_frac.get(), "Fractional", "l"); - leg->AddEntry(heigen_cumulative.get(), "Cumulative", "l"); + leg->AddEntry(heigen_values, "Absolute", "l"); + leg->AddEntry(heigen_frac, "Fractional", "l"); + leg->AddEntry(heigen_cumulative, "Cumulative", "l"); leg->SetLineColor(0); leg->SetLineStyle(0); @@ -525,61 +517,88 @@ void PCAHandler::DebugPCA(const double sum, TMatrixD temp, TMatrixDSym submat, i c1->Print("Debug_PCA.pdf"); c1->SetRightMargin(0.15); c1->SetLogy(0); + delete EigenLine; + delete leg; + delete text; + delete heigen_values; + delete heigen_frac; + delete heigen_cumulative; heigen_vectors->SetMarkerSize(0.2); minz = heigen_vectors->GetMinimum(); - if (fabs(0-maxz)>fabs(0-minz)) heigen_vectors->GetZaxis()->SetRangeUser(0-fabs(0-maxz),0+fabs(0-maxz)); - else heigen_vectors->GetZaxis()->SetRangeUser(0-fabs(0-minz),0+fabs(0-minz)); - if(PlotText) heigen_vectors->Draw("COLZ TEXT"); - else heigen_vectors->Draw("COLZ"); - - auto Eigen_Line = std::make_unique(0, nKeptPCApars, LastPCAdpar - FirstPCAdpar, nKeptPCApars); + if (fabs(0 - maxz) > fabs(0 - minz)) + heigen_vectors->GetZaxis()->SetRangeUser(0 - fabs(0 - maxz), + 0 + fabs(0 - maxz)); + else + heigen_vectors->GetZaxis()->SetRangeUser(0 - fabs(0 - minz), + 0 + fabs(0 - minz)); + if (PlotText) + heigen_vectors->Draw("COLZ TEXT"); + else + heigen_vectors->Draw("COLZ"); + + TLine *Eigen_Line = + new TLine(0, nKeptPCApars, LastPCAdpar - FirstPCAdpar, nKeptPCApars); Eigen_Line->SetLineColor(kGreen); Eigen_Line->SetLineWidth(2); Eigen_Line->SetLineStyle(kDotted); Eigen_Line->Draw("SAME"); c1->Print("Debug_PCA.pdf"); + delete Eigen_Line; SubsetPCA->SetMarkerSize(0.2); minz = SubsetPCA->GetMinimum(); - if (fabs(0-maxz)>fabs(0-minz)) SubsetPCA->GetZaxis()->SetRangeUser(0-fabs(0-maxz),0+fabs(0-maxz)); - else SubsetPCA->GetZaxis()->SetRangeUser(0-fabs(0-minz),0+fabs(0-minz)); - if(PlotText) SubsetPCA->Draw("COLZ TEXT"); - else SubsetPCA->Draw("COLZ"); + if (fabs(0 - maxz) > fabs(0 - minz)) + SubsetPCA->GetZaxis()->SetRangeUser(0 - fabs(0 - maxz), 0 + fabs(0 - maxz)); + else + SubsetPCA->GetZaxis()->SetRangeUser(0 - fabs(0 - minz), 0 + fabs(0 - minz)); + if (PlotText) + SubsetPCA->Draw("COLZ TEXT"); + else + SubsetPCA->Draw("COLZ"); c1->Print("Debug_PCA.pdf"); delete SubsetPCA; hTransferMat->SetMarkerSize(0.15); minz = hTransferMat->GetMinimum(); - if (fabs(0-maxz)>fabs(0-minz)) hTransferMat->GetZaxis()->SetRangeUser(0-fabs(0-maxz),0+fabs(0-maxz)); - else hTransferMat->GetZaxis()->SetRangeUser(0-fabs(0-minz),0+fabs(0-minz)); - if(PlotText) hTransferMat->Draw("COLZ TEXT"); - else hTransferMat->Draw("COLZ"); + if (fabs(0 - maxz) > fabs(0 - minz)) + hTransferMat->GetZaxis()->SetRangeUser(0 - fabs(0 - maxz), + 0 + fabs(0 - maxz)); + else + hTransferMat->GetZaxis()->SetRangeUser(0 - fabs(0 - minz), + 0 + fabs(0 - minz)); + if (PlotText) + hTransferMat->Draw("COLZ TEXT"); + else + hTransferMat->Draw("COLZ"); c1->Print("Debug_PCA.pdf"); delete hTransferMat; hTransferMatT->SetMarkerSize(0.15); minz = hTransferMatT->GetMinimum(); - if (fabs(0-maxz)>fabs(0-minz)) hTransferMatT->GetZaxis()->SetRangeUser(0-fabs(0-maxz),0+fabs(0-maxz)); - else hTransferMatT->GetZaxis()->SetRangeUser(0-fabs(0-minz),0+fabs(0-minz)); - if(PlotText) hTransferMatT->Draw("COLZ TEXT"); - else hTransferMatT->Draw("COLZ"); - c1->Print( "Debug_PCA.pdf"); + if (fabs(0 - maxz) > fabs(0 - minz)) + hTransferMatT->GetZaxis()->SetRangeUser(0 - fabs(0 - maxz), + 0 + fabs(0 - maxz)); + else + hTransferMatT->GetZaxis()->SetRangeUser(0 - fabs(0 - minz), + 0 + fabs(0 - minz)); + if (PlotText) + hTransferMatT->Draw("COLZ TEXT"); + else + hTransferMatT->Draw("COLZ"); + c1->Print("Debug_PCA.pdf"); delete hTransferMatT; - - //KS: Crosscheck against Eigen library - #if DEBUG_PCA == 2 +// KS: Crosscheck against Eigen library +#if DEBUG_PCA == 2 Eigen::MatrixXd Submat_Eigen(submat.GetNrows(), submat.GetNcols()); - #ifdef MULTITHREAD - #pragma omp parallel for - #endif - for(int i = 0; i < submat.GetNrows(); i++) - { - for(int j = 0; j < submat.GetNcols(); j++) - { - Submat_Eigen(i,j) = (submat)(i,j); +#ifdef MULTITHREAD +#pragma omp parallel for +#endif + for (int i = 0; i < submat.GetNrows(); i++) { + for (int j = 0; j < submat.GetNcols(); j++) { + Submat_Eigen(i, j) = (submat)(i, j); } } Eigen::EigenSolver EigenSolver; @@ -588,35 +607,40 @@ void PCAHandler::DebugPCA(const double sum, TMatrixD temp, TMatrixDSym submat, i Eigen::MatrixXd eigen_vect = EigenSolver.eigenvectors().real(); std::vector> eigen_vectors_and_values; double Sum_Eigen = 0; - for(int i = 0; i < eigen_val.size(); i++) - { - std::tuple vec_and_val(eigen_val[i], eigen_vect.row(i)); + for (int i = 0; i < eigen_val.size(); i++) { + std::tuple vec_and_val(eigen_val[i], + eigen_vect.row(i)); eigen_vectors_and_values.push_back(vec_and_val); Sum_Eigen += eigen_val[i]; } std::sort(eigen_vectors_and_values.begin(), eigen_vectors_and_values.end(), - [&](const std::tuple& a, const std::tuple& b) -> bool - { return std::get<0>(a) > std::get<0>(b); } ); + [&](const std::tuple &a, + const std::tuple &b) -> bool { + return std::get<0>(a) > std::get<0>(b); + }); int index = 0; - for(auto const vect : eigen_vectors_and_values) - { + for (auto const vect : eigen_vectors_and_values) { eigen_val(index) = std::get<0>(vect); eigen_vect.row(index) = std::get<1>(vect); index++; } - TH1D* heigen_values_Eigen = new TH1D("eig_values", "Eigen Values", eigen_val.size(), 0.0, eigen_val.size()); - TH1D* heigen_cumulative_Eigen = new TH1D("eig_cumulative", "heigen_cumulative", eigen_val.size(), 0.0, eigen_val.size()); - TH1D* heigen_frac_Eigen = new TH1D("eig_fractional", "heigen_fractional", eigen_val.size(), 0.0, eigen_val.size()); + TH1D *heigen_values_Eigen = new TH1D("eig_values", "Eigen Values", + eigen_val.size(), 0.0, eigen_val.size()); + TH1D *heigen_cumulative_Eigen = + new TH1D("eig_cumulative", "heigen_cumulative", eigen_val.size(), 0.0, + eigen_val.size()); + TH1D *heigen_frac_Eigen = new TH1D("eig_fractional", "heigen_fractional", + eigen_val.size(), 0.0, eigen_val.size()); heigen_values_Eigen->GetXaxis()->SetTitle("Eigen Vector"); heigen_values_Eigen->GetYaxis()->SetTitle("Eigen Value"); double Cumulative_Eigen = 0; - for(int i = 0; i < eigen_val.size(); i++) - { - heigen_values_Eigen->SetBinContent(i+1, eigen_val(i)); - heigen_cumulative_Eigen->SetBinContent(i+1, eigen_val(i)/sum + Cumulative_Eigen); - heigen_frac_Eigen->SetBinContent(i+1, eigen_val(i)/sum); - Cumulative_Eigen += eigen_val(i)/sum; + for (int i = 0; i < eigen_val.size(); i++) { + heigen_values_Eigen->SetBinContent(i + 1, eigen_val(i)); + heigen_cumulative_Eigen->SetBinContent(i + 1, eigen_val(i) / sum + + Cumulative_Eigen); + heigen_frac_Eigen->SetBinContent(i + 1, eigen_val(i) / sum); + Cumulative_Eigen += eigen_val(i) / sum; } heigen_values_Eigen->SetLineColor(kRed); heigen_values_Eigen->SetLineWidth(2); @@ -626,12 +650,13 @@ void PCAHandler::DebugPCA(const double sum, TMatrixD temp, TMatrixDSym submat, i heigen_frac_Eigen->SetLineWidth(2); c1->SetLogy(); - heigen_values_Eigen->SetMaximum(heigen_cumulative_Eigen->GetMaximum()+heigen_cumulative_Eigen->GetMaximum()*0.4); + heigen_values_Eigen->SetMaximum(heigen_cumulative_Eigen->GetMaximum() + + heigen_cumulative_Eigen->GetMaximum() * 0.4); heigen_values_Eigen->Draw(); heigen_cumulative_Eigen->Draw("SAME"); heigen_frac_Eigen->Draw("SAME"); - auto leg_Eigen = std::make_unique(0.2, 0.2, 0.6, 0.5); + TLegend *leg_Eigen = new TLegend(0.2, 0.2, 0.6, 0.5); leg_Eigen->SetTextSize(0.04); leg_Eigen->AddEntry(heigen_values_Eigen, "Absolute", "l"); leg_Eigen->AddEntry(heigen_frac_Eigen, "Fractional", "l"); @@ -648,44 +673,57 @@ void PCAHandler::DebugPCA(const double sum, TMatrixD temp, TMatrixDSym submat, i delete heigen_values_Eigen; delete heigen_cumulative_Eigen; delete heigen_frac_Eigen; + delete leg_Eigen; - TH2D* heigen_vectors_Eigen = new TH2D("Eigen_Vectors", "Eigen_Vectors", eigen_val.size(), 0.0, eigen_val.size(), eigen_val.size(), 0.0, eigen_val.size()); + TH2D *heigen_vectors_Eigen = + new TH2D("Eigen_Vectors", "Eigen_Vectors", eigen_val.size(), 0.0, + eigen_val.size(), eigen_val.size(), 0.0, eigen_val.size()); - for(int i = 0; i < eigen_val.size(); i++) - { - for(int j = 0; j < eigen_val.size(); j++) - { - //KS: +1 because there is offset in histogram relative to TMatrix - heigen_vectors_Eigen->SetBinContent(i+1,j+1, eigen_vect(i,j)); + for (int i = 0; i < eigen_val.size(); i++) { + for (int j = 0; j < eigen_val.size(); j++) { + // KS: +1 because there is offset in histogram relative to TMatrix + heigen_vectors_Eigen->SetBinContent(i + 1, j + 1, eigen_vect(i, j)); } } heigen_vectors_Eigen->GetXaxis()->SetTitle("Parameter in Normal Base"); heigen_vectors_Eigen->GetYaxis()->SetTitle("Parameter in Decomposed Base"); heigen_vectors_Eigen->SetMarkerSize(0.15); minz = heigen_vectors_Eigen->GetMinimum(); - if (fabs(0-maxz)>fabs(0-minz)) heigen_vectors_Eigen->GetZaxis()->SetRangeUser(0-fabs(0-maxz),0+fabs(0-maxz)); - else heigen_vectors_Eigen->GetZaxis()->SetRangeUser(0-fabs(0-minz),0+fabs(0-minz)); - - if(PlotText) heigen_vectors_Eigen->Draw("COLZ TEXT"); - else heigen_vectors_Eigen->Draw("COLZ"); - c1->Print( "Debug_PCA.pdf"); + if (fabs(0 - maxz) > fabs(0 - minz)) + heigen_vectors_Eigen->GetZaxis()->SetRangeUser(0 - fabs(0 - maxz), + 0 + fabs(0 - maxz)); + else + heigen_vectors_Eigen->GetZaxis()->SetRangeUser(0 - fabs(0 - minz), + 0 + fabs(0 - minz)); + + if (PlotText) + heigen_vectors_Eigen->Draw("COLZ TEXT"); + else + heigen_vectors_Eigen->Draw("COLZ"); + c1->Print("Debug_PCA.pdf"); heigen_vectors->SetTitle("ROOT minus Eigen"); heigen_vectors->Add(heigen_vectors_Eigen, -1.); minz = heigen_vectors->GetMinimum(); - if (fabs(0-maxz)>fabs(0-minz)) heigen_vectors->GetZaxis()->SetRangeUser(0-fabs(0-maxz),0+fabs(0-maxz)); - else heigen_vectors->GetZaxis()->SetRangeUser(0-fabs(0-minz),0+fabs(0-minz)); - if(PlotText) heigen_vectors->Draw("COLZ TEXT"); - else heigen_vectors->Draw("COLZ"); + if (fabs(0 - maxz) > fabs(0 - minz)) + heigen_vectors->GetZaxis()->SetRangeUser(0 - fabs(0 - maxz), + 0 + fabs(0 - maxz)); + else + heigen_vectors->GetZaxis()->SetRangeUser(0 - fabs(0 - minz), + 0 + fabs(0 - minz)); + if (PlotText) + heigen_vectors->Draw("COLZ TEXT"); + else + heigen_vectors->Draw("COLZ"); c1->Print("Debug_PCA.pdf"); delete heigen_vectors_Eigen; - #endif // end if Eigen enabled +#endif // end if Eigen enabled delete heigen_vectors; c1->Print("Debug_PCA.pdf]"); + delete c1; PCA_Debug->Close(); delete PCA_Debug; - gErrorIgnoreLevel = originalErrorWarning; } #endif diff --git a/Parameters/PCAHandler.h b/Parameters/PCAHandler.h index 9659455d2..89d689788 100644 --- a/Parameters/PCAHandler.h +++ b/Parameters/PCAHandler.h @@ -60,33 +60,27 @@ class PCAHandler{ public: /// @brief Constructor - PCAHandler(); + PCAHandler(){}; /// @brief Destructor - virtual ~PCAHandler(); + virtual ~PCAHandler(){}; /// @brief KS: Print info about PCA parameters void Print() const; /// @brief Retrieve number of parameters in PCA base int GetNumberPCAedParameters() const { return NumParPCA; } - /// @brief KS: Setup pointers to current and proposed parameter value which we need to convert them to PCA base each step - /// @param fCurr_Val pointer to current position of parameter - /// @param fProp_Val pointer to proposed position of parameter - void SetupPointers(std::vector* fCurr_Val, - std::vector* fProp_Val); - /// @brief CW: Calculate eigen values, prepare transition matrices and remove param based on defined threshold /// @param CovMatrix Symmetric covariance matrix used for eigen decomposition. /// @param firstPCAd Index of the first PCA component to include. /// @param lastPCAd Index of the last PCA component to include. /// @param eigen_thresh Threshold for eigenvalues below which parameters are discarded. /// @param _fNumPar Total number of parameters in the original (non-PCA) basis. - void ConstructPCA(TMatrixDSym * CovMatrix, const int firstPCAd, const int lastPCAd, - const double eigen_thresh, const int _fNumPar); + void ConstructPCA(Eigen::MatrixXd const & CovMatrix, const int firstPCAd, const int lastPCAd, + const double eigen_thresh); /// @brief Transfer param values from normal base to PCA base - void TransferToPCA(); + void SetUnrotatedParameterValues(Eigen::VectorXd const &vals); /// @brief Transfer param values from PCA base to normal base void TransferToParam(); @@ -100,7 +94,7 @@ class PCAHandler{ const std::vector& fUpBound, int _fNumPar); - /// @brief Accepted this step + /// @brief Accept this step void AcceptStep() _noexcept_; /// @brief Use Cholesky throw matrix for better step proposal void CorrelateSteps(const std::vector& IndivStepScale, @@ -226,8 +220,6 @@ class PCAHandler{ #endif private: - /// @brief KS: Make sure decomposed matrix isn't correlated with undecomposed - void SanitisePCA(TMatrixDSym* CovMatrix); /// Prefit value for PCA params std::vector _fPreFitValuePCA; diff --git a/Parameters/ParameterHandlerBase.cpp b/Parameters/ParameterHandlerBase.cpp index a1e82da2e..ae9745443 100644 --- a/Parameters/ParameterHandlerBase.cpp +++ b/Parameters/ParameterHandlerBase.cpp @@ -1,301 +1,271 @@ #include "Parameters/ParameterHandlerBase.h" -#include "regex" +#include + +ParameterHandlerBase::ParameterHandlerBase() + : pca.enabled{false}, +special_proposal.enabled{false}, rng.gaus(0), settings.use_adaptive{false}, +settings.PrintLength{35} {} // ******************************************** -ParameterHandlerBase::ParameterHandlerBase(std::string name, std::string file, double threshold, int FirstPCA, int LastPCA) - : inputFile(file), pca(false) { -// ******************************************** - MACH3LOG_DEBUG("Constructing instance of ParameterHandler"); - doSpecialStepProposal = false; - if (threshold < 0 || threshold >= 1) { - MACH3LOG_INFO("NOTE: {} {}", name, file); - MACH3LOG_INFO("Principal component analysis but given the threshold for the principal components to be less than 0, or greater than (or equal to) 1. This will not work"); - MACH3LOG_INFO("Please specify a number between 0 and 1"); - MACH3LOG_INFO("You specified: "); - MACH3LOG_INFO("Am instead calling the usual non-PCA constructor..."); - pca = false; - } - Init(name, file); +ParameterHandlerBase::ParameterHandlerBase(std::string const &name, + std::string const &inputFile) + : ParameterHandlerBase() { + // ******************************************** + MACH3LOG_DEBUG("Constructing instance of ParameterHandler, named: {}", name); - // Call the innocent helper function - if (pca) ConstructPCA(threshold, FirstPCA, LastPCA); + SetName(name); + config.inputFile = file; } + // ******************************************** -ParameterHandlerBase::ParameterHandlerBase(const std::vector& YAMLFile, std::string name, double threshold, int FirstPCA, int LastPCA) - : inputFile(YAMLFile[0].c_str()), matrixName(name), pca(true) { -// ******************************************** - MACH3LOG_INFO("Constructing instance of ParameterHandler using"); - doSpecialStepProposal = false; - for(unsigned int i = 0; i < YAMLFile.size(); i++) - { - MACH3LOG_INFO("{}", YAMLFile[i]); - } - MACH3LOG_INFO("as an input"); +ParameterHandlerBase::ParameterHandlerBase(std::string name, std::string file, + double threshold, int FirstPCA, + int LastPCA) + : ParameterHandlerBase(name, file) { + // ******************************************** - if (threshold < 0 || threshold >= 1) { - MACH3LOG_INFO("Principal component analysis but given the threshold for the principal components to be less than 0, or greater than (or equal to) 1. This will not work"); - MACH3LOG_INFO("Please specify a number between 0 and 1"); - MACH3LOG_INFO("You specified: "); - MACH3LOG_INFO("Am instead calling the usual non-PCA constructor..."); - pca = false; + // Set the covariance matrix from input ROOT file (e.g. flux, ND280, NIWG) + TFile infile(file.c_str(), "READ"); + if (infile.IsZombie()) { + MACH3LOG_ERROR("Could not open input covariance ROOT file {} !!!", file); + MACH3LOG_ERROR("Was about to retrieve matrix with name {}", name); + throw MaCh3Exception(__FILE__, __LINE__); } - Init(YAMLFile); - // Call the innocent helper function - if (pca) ConstructPCA(threshold, FirstPCA, LastPCA); -} + auto *CovMat = infile.Get(name.c_str()); -// ******************************************** -//Destructor -ParameterHandlerBase::~ParameterHandlerBase(){ -// ******************************************** - delete[] randParams; - delete[] corr_throw; - - if (covMatrix != nullptr) delete covMatrix; - if (invCovMatrix != nullptr) delete invCovMatrix; - if (throwMatrix != nullptr) delete throwMatrix; - for(int i = 0; i < _fNumPar; i++) { - delete[] throwMatrixCholDecomp[i]; + if (!CovMat) { + MACH3LOG_ERROR("Could not find covariance matrix name {} in file {}", name, + file); + MACH3LOG_ERROR("Are you really sure {} exists in the file?", name); + throw MaCh3Exception(__FILE__, __LINE__); } - delete[] throwMatrixCholDecomp; -} -// ******************************************** -void ParameterHandlerBase::ConstructPCA(const double eigen_threshold, int FirstPCAdpar, int LastPCAdpar) { -// ******************************************** - if(AdaptiveHandler) { - MACH3LOG_ERROR("Adaption has been enabled and now trying to enable PCA. Right now both configuration don't work with each other"); - throw MaCh3Exception(__FILE__ , __LINE__ ); - } + params.covariance = M3::MakePosDef(M3::ROOTToEigen(*CovMat)); - PCAObj = std::make_unique(); - //Check whether first and last pcadpar are set and if not just PCA everything - if(FirstPCAdpar == -999 || LastPCAdpar == -999){ - if(FirstPCAdpar == -999 && LastPCAdpar == -999){ - FirstPCAdpar = 0; - LastPCAdpar = covMatrix->GetNrows()-1; - } - else{ - MACH3LOG_ERROR("You must either leave FirstPCAdpar and LastPCAdpar at -999 or set them both to something"); - throw MaCh3Exception(__FILE__ , __LINE__ ); - } + for (int iThread = 0; iThread < M3::GetNThreads(); iThread++) { + random_number.emplace_back(0); } - PCAObj->ConstructPCA(covMatrix, FirstPCAdpar, LastPCAdpar, eigen_threshold, _fNumPar); - PCAObj->SetupPointers(&_fCurrVal, &_fPropVal); - // Make a note that we have now done PCA - pca = true; + MACH3LOG_INFO("Created covariance matrix named: {}", GetName()); + MACH3LOG_INFO("from file: {}", file); } -// ******************************************** -void ParameterHandlerBase::Init(const std::string& name, const std::string& file) { -// ******************************************** - // Set the covariance matrix from input ROOT file (e.g. flux, ND280, NIWG) - TFile *infile = new TFile(file.c_str(), "READ"); - if (infile->IsZombie()) { - MACH3LOG_ERROR("Could not open input covariance ROOT file {} !!!", file); - MACH3LOG_ERROR("Was about to retrieve matrix with name {}", name); - throw MaCh3Exception(__FILE__ , __LINE__ ); +void ParameterHandlerBase::AddParameters(std::vector const ¶ms) { + + for (auto const &p : params) { + params.name.push_back(p.name); + params.fancy_name.push_back(p.fancy_name); + params.samples.push_back(p.affected_samples); } - TMatrixDSym *CovMat = static_cast(infile->Get(name.c_str())); + size_t new_block_start = params.prefit.size(); - if (!CovMat) { - MACH3LOG_ERROR("Could not find covariance matrix name {} in file {}", name, file); - MACH3LOG_ERROR("Are you really sure {} exists in the file?", name); - throw MaCh3Exception(__FILE__ , __LINE__ ); - } + params.prefit.conservativeResize(params.name.size()); + params.error.conservativeResize(params.name.size()); + params.lowbound.conservativeResize(params.name.size()); + params.upbound.conservativeResize(params.name.size()); + params.flatprior.conservativeResize(params.name.size()); + params.fixed.conservativeResize(params.name.size()); + steps.scale.conservativeResize(params.name.size()); - PrintLength = 35; + params.covariance.conservativeResize(params.name.size(), params.name.size()); - const int nThreads = M3::GetNThreads(); - //KS: set Random numbers for each thread so each thread has different seed - //or for one thread if without MULTITHREAD - random_number.reserve(nThreads); - for (int iThread = 0; iThread < nThreads; iThread++) { - random_number.emplace_back(std::make_unique(0)); - } - // Not using adaptive by default - use_adaptive = false; - // Set the covariance matrix - _fNumPar = CovMat->GetNrows(); - - InvertCovMatrix.resize(_fNumPar, std::vector(_fNumPar, 0.0)); - throwMatrixCholDecomp = new double*[_fNumPar](); - // Set the defaults to true - for(int i = 0; i < _fNumPar; i++) { - throwMatrixCholDecomp[i] = new double[_fNumPar](); - for (int j = 0; j < _fNumPar; j++) { - throwMatrixCholDecomp[i][j] = 0.; - } + for (size_t i = 0; i < params.size(); ++i) { + params.prefit[new_block_start + i] = params[i].prefit; + params.error[new_block_start + i] = params[i].error; + params.lowbound[new_block_start + i] = params[i].bounds[0]; + params.upbound[new_block_start + i] = params[i].bounds[1]; + params.flatprior[new_block_start + i] = params[i].flatprior; + params.fixed[new_block_start + i] = false; + steps.scale[new_block_start + i] = params[i].stepscale; + params.covariance(new_block_start + i, new_block_start + i) = + params[i].error * params[i].error; } - SetName(name); - MakePosDef(CovMat); - SetCovMatrix(CovMat); - if (_fNumPar <= 0) { - MACH3LOG_CRITICAL("Covariance matrix {} has {} entries!", GetName(), _fNumPar); - throw MaCh3Exception(__FILE__ , __LINE__ ); + + params.inv_covariance = MatrixXd(0); +} + +void ParameterHandlerBase::SetParameterCorrelation(int pidi, int pidj, + double corr) { + if (pidi == pidj) { + MACH3LOG_ERROR( + "AddParameterCorrelation cannot be used to set covariance " + "matrix diagonal elements: ({0},{0}) attempted to be set to {1}", + pidi, corr); + throw MaCh3Exception(__FILE__, __LINE__); } - ReserveMemory(_fNumPar); + params.covariance(pidi, pidj) = + corr * + std::sqrt(params.covariance(pidi, pidi) * params.covariance(pidj, pidj)); + params.covariance(pidj, pidi) = params.covariance(pidi, pidj); - infile->Close(); + params.inv_covariance = MatrixXd(0); +} - MACH3LOG_INFO("Created covariance matrix named: {}", GetName()); - MACH3LOG_INFO("from file: {}", file); - delete infile; +void ParameterHandlerBase::SetParameterAllCorrelations( + int paramid, std::map const &correlations) { + + params.covariance.row(paramid) = + Eigen::VectorXd::Zeros(params.covariance.rows()); + params.covariance.col(paramid) = + Eigen::VectorXd::Zeros(params.covariance.rows()); + + params.covariance(paramid, paramid) = + params.error[paramid] * params.error[paramid]; + + for (auto const &[other_name, corr] : correlations) { + SetParameterCorrelation(paramid, GetParIndex(other_name), corr); + } } // ******************************************** -// ETA An init function for the YAML constructor -// All you really need from the YAML file is the number of Systematics -void ParameterHandlerBase::Init(const std::vector& YAMLFile) { -// ******************************************** - _fYAMLDoc["Systematics"] = YAML::Node(YAML::NodeType::Sequence); - for(unsigned int i = 0; i < YAMLFile.size(); i++) - { - YAML::Node YAMLDocTemp = M3OpenConfig(YAMLFile[i]); - for (const auto& item : YAMLDocTemp["Systematics"]) { - _fYAMLDoc["Systematics"].push_back(item); - } - } +ParameterHandlerBase::ParameterHandlerBase( + const std::vector &YAMLFiles, std::string name, + double threshold, int FirstPCA, int LastPCA) + : ParameterHandlerBase(name, YAMLFiles[0]) { + // ******************************************** - const int nThreads = M3::GetNThreads(); - //KS: set Random numbers for each thread so each thread has different seed - //or for one thread if without MULTITHREAD - random_number.reserve(nThreads); - for (int iThread = 0; iThread < nThreads; iThread++) { - random_number.emplace_back(std::make_unique(0)); + MACH3LOG_INFO("Constructing instance of ParameterHandler using: "); + for (auto const &yf : YAMLFile) { + MACH3LOG_INFO(" {}", yf); } - PrintLength = 35; - - // Set the covariance matrix - _fNumPar = int(_fYAMLDoc["Systematics"].size()); + MACH3LOG_INFO("as an input."); - use_adaptive = false; + settings.pca = true; + if (threshold < 0 || threshold >= 1) { + MACH3LOG_INFO("Principal component analysis but given the threshold for " + "the principal components to be less than 0, or greater than " + "(or equal to) 1. This will not work"); + MACH3LOG_INFO("Please specify a number between 0 and 1"); + MACH3LOG_INFO("You specified: {}", threshold); + MACH3LOG_INFO("Disabling PCA..."); + settings.pca = false; + } - InvertCovMatrix.resize(_fNumPar, std::vector(_fNumPar, 0.0)); - throwMatrixCholDecomp = new double*[_fNumPar](); - for(int i = 0; i < _fNumPar; i++) { - throwMatrixCholDecomp[i] = new double[_fNumPar](); - for (int j = 0; j < _fNumPar; j++) { - throwMatrixCholDecomp[i][j] = 0.; + config.YAMLDoc["Systematics"] = YAML::Node(YAML::NodeType::Sequence); + for (auto const &yfn : YAMLFiles) { + for (const auto &[idx, node] : M3::OpenConfig(yf)["Systematics"]) { + config.YAMLDoc["Systematics"].push_back(node); } } - ReserveMemory(_fNumPar); - TMatrixDSym* _fCovMatrix = new TMatrixDSym(_fNumPar); - int i = 0; - std::vector> Correlations(_fNumPar); - std::map CorrNamesMap; - - //ETA - read in the systematics. Would be good to add in some checks to make sure - //that there are the correct number of entries i.e. are the _fNumPar for Names, - //PreFitValues etc etc. - for (auto const ¶m : _fYAMLDoc["Systematics"]) - { - _fFancyNames[i] = Get(param["Systematic"]["Names"]["FancyName"], __FILE__ , __LINE__); - _fPreFitValue[i] = Get(param["Systematic"]["ParameterValues"]["PreFitValue"], __FILE__ , __LINE__); - _fIndivStepScale[i] = Get(param["Systematic"]["StepScale"]["MCMC"], __FILE__ , __LINE__); - _fError[i] = Get(param["Systematic"]["Error"], __FILE__ , __LINE__); - _fSampleNames[i] = GetFromManager>(param["Systematic"]["SampleNames"], {}, __FILE__, __LINE__); - if(_fError[i] <= 0) { - MACH3LOG_ERROR("Error for param {}({}) is negative and equal to {}", _fFancyNames[i], i, _fError[i]); - throw MaCh3Exception(__FILE__ , __LINE__ ); - } - //ETA - a bit of a fudge but works - auto TempBoundsVec = GetBounds(param["Systematic"]["ParameterBounds"]); - _fLowBound[i] = TempBoundsVec[0]; - _fUpBound[i] = TempBoundsVec[1]; + std::map> parameter_correlations; - //ETA - now for parameters which are optional and have default values - _fFlatPrior[i] = GetFromManager(param["Systematic"]["FlatPrior"], false, __FILE__ , __LINE__); + std::vector param_infos; - // Allow to fix param, this setting should be used only for params which are permanently fixed like baseline, please use global config for fixing param more flexibly - if(GetFromManager(param["Systematic"]["FixParam"], false, __FILE__ , __LINE__)) { - ToggleFixParameter(_fFancyNames[i]); - } + // ETA - read in the systematics. + for (auto const &[idx, node] : config.YAMLDoc["Systematics"]) { + size_t param_id = param_infos.size(); + + auto const &pardef = node["Systematic"]; + + auto fancy_name = GetFromManager(pardef["Names"]["FancyName"], + __FILE__, __LINE__); + auto prefit = GetFromManager( + pardef["ParameterValues"]["PreFitValue"], __FILE__, __LINE__); - if(param["Systematic"]["SpecialProposal"]) { - EnableSpecialProposal(param["Systematic"]["SpecialProposal"], i); + auto error = GetFromManager(pardef["Error"], __FILE__, __LINE__); + + if (error <= 0) { + MACH3LOG_ERROR("Error for param {}({}) is negative and equal to {}", + fancy_name, param_id, error); + throw MaCh3Exception(__FILE__, __LINE__); } - //Fill the map to get the correlations later as well - CorrNamesMap[param["Systematic"]["Names"]["FancyName"].as()]=i; + auto stepscale = + GetFromManager(pardef["StepScale"]["MCMC"], __FILE__, __LINE__); - //Also loop through the correlations - if(param["Systematic"]["Correlations"]) { - for(unsigned int Corr_i = 0; Corr_i < param["Systematic"]["Correlations"].size(); ++Corr_i){ - for (YAML::const_iterator it = param["Systematic"]["Correlations"][Corr_i].begin(); it!=param["Systematic"]["Correlations"][Corr_i].end();++it) { - Correlations[i][it->first.as()] = it->second.as(); - } - } + auto bounds = GetBounds(pardef["ParameterBounds"]); + + auto flatprior = + GetFromManager(pardef["FlatPrior"], false, __FILE__, __LINE__); + + auto samplenames = GetFromManager>( + pardef["SampleNames"], {}, __FILE__, __LINE__); + + // Allow to fix param, this setting should be used only for params which are + // permanently fixed like baseline, please use global config for fixing + // param more flexibly + auto fixed = + GetFromManager(pardef["FixParam"], false, __FILE__, __LINE__); + + if (pardef["SpecialProposal"]) { + special_proposals.push_back(param_id); + EnableSpecialProposal(pardef["SpecialProposal"]); } - i++; - } // end loop over para - if(i != _fNumPar) { - MACH3LOG_CRITICAL("Inconsistent number of params in Yaml {} vs {}, this indicate wrong syntax", i, i, _fNumPar); - throw MaCh3Exception(__FILE__ , __LINE__ ); - } - // ETA Now that we've been through all systematic let's fill the covmatrix - //This makes the root TCov from YAML - for(int j = 0; j < _fNumPar; j++) { - (*_fCovMatrix)(j, j) = _fError[j]*_fError[j]; - //Get the map of parameter name to correlation from the Correlations object - for (auto const& pair : Correlations[j]) { - auto const& key = pair.first; - auto const& val = pair.second; - int index = -1; - //If you found the parameter name then get the index - if (CorrNamesMap.find(key) != CorrNamesMap.end()) { - index = CorrNamesMap[key]; - } else { - MACH3LOG_ERROR("Parameter {} not in list! Check your spelling?", key); - throw MaCh3Exception(__FILE__ , __LINE__ ); - } - double Corr1 = val; - double Corr2 = 0; - if(Correlations[index].find(_fFancyNames[j]) != Correlations[index].end()) { - Corr2 = Correlations[index][_fFancyNames[j]]; - //Do they agree to better than float precision? - if(std::abs(Corr2 - Corr1) > FLT_EPSILON) { - MACH3LOG_ERROR("Correlations are not equal between {} and {}", _fFancyNames[j], key); - MACH3LOG_ERROR("Got : {} and {}", Corr2, Corr1); - throw MaCh3Exception(__FILE__ , __LINE__ ); + + if (pardef["Correlations"]) { + for (auto const &[_, it] : pardef["Correlations"]) { + for (auto const &[key, corr] : it) { + parameter_correlations[param_id][key.as()] = + corr.as(); } - } else { - MACH3LOG_ERROR("Correlation does not appear reciprocally between {} and {}", _fFancyNames[j], key); - throw MaCh3Exception(__FILE__ , __LINE__ ); } - (*_fCovMatrix)(j, index)= (*_fCovMatrix)(index, j) = Corr1*_fError[j]*_fError[index]; } + + param_infos.emplace_back({fancy_name, + fancy_name, + prefit, + error, + stepscale, + {bounds[0], bounds[1]}, + flatprior, + samplenames}); } - //Now make positive definite - MakePosDef(_fCovMatrix); - SetCovMatrix(_fCovMatrix); + AddParameters(param_infos); - if (_fNumPar <= 0) { - MACH3LOG_ERROR("ParameterHandler object has {} systematics!", _fNumPar); - throw MaCh3Exception(__FILE__ , __LINE__ ); + for (auto const &[paramid, correlations] : parameter_correlations) { + SetParameterCorrelations(paramid, correlations); } - Tunes = std::make_unique(_fYAMLDoc["Systematics"]); + params.covariance = M3::MakePosDef(params.covariance); + + Tunes = ParameterTunes(config.YAMLDoc["Systematics"]); MACH3LOG_INFO("Created covariance matrix from files: "); - for(const auto &file : YAMLFile){ + for (const auto &file : YAMLFile) { MACH3LOG_INFO("{} ", file); } MACH3LOG_INFO("----------------"); - MACH3LOG_INFO("Found {} systematics parameters in total", _fNumPar); + MACH3LOG_INFO("Found {} systematics parameters in total", + params.prefit.size()); MACH3LOG_INFO("----------------"); } // ******************************************** -void ParameterHandlerBase::EnableSpecialProposal(const YAML::Node& param, const int Index){ +void ParameterHandlerBase::ConstructPCA(const double eigen_threshold, + int FirstPCAdpar, int LastPCAdpar) { + // ******************************************** + if (AdaptiveHandler) { + MACH3LOG_ERROR("Adaption has been enabled and now trying to enable PCA. " + "Right now both configuration don't work with each other"); + throw MaCh3Exception(__FILE__, __LINE__); + } + + // Check whether first and last pcadpar are set and if not just PCA everything + if (FirstPCAdpar == -999 || LastPCAdpar == -999) { + if (FirstPCAdpar == -999 && LastPCAdpar == -999) { + FirstPCAdpar = 0; + LastPCAdpar = covMatrix->GetNrows() - 1; + } else { + MACH3LOG_ERROR("You must either leave FirstPCAdpar and LastPCAdpar at " + "-999 or set them both to something"); + throw MaCh3Exception(__FILE__, __LINE__); + } + } + + PCAObj.ConstructPCA(params.covariance, FirstPCAdpar, LastPCAdpar, + eigen_threshold); +} + // ******************************************** - doSpecialStepProposal = true; +void ParameterHandlerBase::EnableSpecialProposal(const YAML::Node ¶m, + const int Index) { + // ******************************************** + special_proposal.enabled = true; bool CircEnabled = false; bool FlipEnabled = false; @@ -309,54 +279,72 @@ void ParameterHandlerBase::EnableSpecialProposal(const YAML::Node& param, const } if (!CircEnabled && !FlipEnabled) { - MACH3LOG_ERROR("None of Special Proposal were enabled even though param {}, has SpecialProposal entry in Yaml", GetParFancyName(Index)); + MACH3LOG_ERROR("None of Special Proposal were enabled even though param " + "{}, has SpecialProposal entry in Yaml", + GetParFancyName(Index)); throw MaCh3Exception(__FILE__, __LINE__); } if (CircEnabled) { - CircularBoundsIndex.push_back(Index); - CircularBoundsValues.push_back(Get>(param["CircularBounds"], __FILE__, __LINE__)); - MACH3LOG_INFO("Enabling CircularBounds for parameter {} with range [{}, {}]", - GetParFancyName(Index), - CircularBoundsValues.back().first, - CircularBoundsValues.back().second); - // KS: Make sure circular bounds are within physical bounds. If we are outside of physics bound MCMC will never explore such phase space region - if (CircularBoundsValues.back().first < _fLowBound.at(Index) || CircularBoundsValues.back().second > _fUpBound.at(Index)) { - MACH3LOG_ERROR("Circular bounds [{}, {}] for parameter {} exceed physical bounds [{}, {}]", - CircularBoundsValues.back().first, CircularBoundsValues.back().second, - GetParFancyName(Index), - _fLowBound.at(Index), _fUpBound.at(Index)); + special_proposal.CircularBoundsIndex.push_back(Index); + special_proposal.CircularBoundsValues.push_back( + Get>(param["CircularBounds"], __FILE__, + __LINE__)); + MACH3LOG_INFO( + "Enabling CircularBounds for parameter {} with range [{}, {}]", + GetParFancyName(Index), + special_proposal.CircularBoundsValues.back().first, + special_proposal.CircularBoundsValues.back().second); + // KS: Make sure circular bounds are within physical bounds. If we are + // outside of physics bound MCMC will never explore such phase space region + if (special_proposal.CircularBoundsValues.back().first < + _fLowBound.at(Index) || + special_proposal.CircularBoundsValues.back().second > + _fUpBound.at(Index)) { + MACH3LOG_ERROR("Circular bounds [{}, {}] for parameter {} exceed " + "physical bounds [{}, {}]", + special_proposal.CircularBoundsValues.back().first, + special_proposal.CircularBoundsValues.back().second, + GetParFancyName(Index), _fLowBound.at(Index), + _fUpBound.at(Index)); throw MaCh3Exception(__FILE__, __LINE__); } } if (FlipEnabled) { - FlipParameterIndex.push_back(Index); - FlipParameterPoint.push_back(Get(param["FlipParameter"], __FILE__, __LINE__)); + special_proposal.FlipParameterIndex.push_back(Index); + special_proposal.FlipParameterPoint.push_back( + Get(param["FlipParameter"], __FILE__, __LINE__)); MACH3LOG_INFO("Enabling Flipping for parameter {} with value {}", GetParFancyName(Index), - FlipParameterPoint.back()); + special_proposal.FlipParameterPoint.back()); } if (CircEnabled && FlipEnabled) { - if (FlipParameterPoint.back() < CircularBoundsValues.back().first || FlipParameterPoint.back() > CircularBoundsValues.back().second) { - MACH3LOG_ERROR("FlipParameter value {} for parameter {} is outside the CircularBounds [{}, {}]", - FlipParameterPoint.back(), GetParFancyName(Index), CircularBoundsValues.back().first, CircularBoundsValues.back().second); + + const double fp = special_proposal.FlipParameterPoint.back(); + const double low = special_proposal.CircularBoundsValues.back().first; + const double high = special_proposal.CircularBoundsValues.back().second; + + if (fp < low || fp > high) { + MACH3LOG_ERROR("FlipParameter value {} for parameter {} is outside the " + "CircularBounds [{}, {}]", + fp, GetParFancyName(Index), low, high); throw MaCh3Exception(__FILE__, __LINE__); } - const double low = CircularBoundsValues.back().first; - const double high = CircularBoundsValues.back().second; - - // Sanity check: ensure flipping any x in [low, high] keeps the result in [low, high] - const double flipped_low = 2 * FlipParameterPoint.back() - low; - const double flipped_high = 2 * FlipParameterPoint.back() - high; + // Sanity check: ensure flipping any x in [low, high] keeps the result in + // [low, high] + const double flipped_low = 2 * fp - low; + const double flipped_high = 2 * fp - high; const double min_flip = std::min(flipped_low, flipped_high); const double max_flip = std::max(flipped_low, flipped_high); if (min_flip < low || max_flip > high) { - MACH3LOG_ERROR("Flipping about point {} for parameter {} would leave circular bounds [{}, {}]", - FlipParameterPoint.back(), GetParFancyName(Index), low, high); + MACH3LOG_ERROR("Flipping about point {} for parameter {} would leave " + "circular bounds [{}, {}]", + special_proposal.FlipParameterPoint.back(), + GetParFancyName(Index), low, high); throw MaCh3Exception(__FILE__, __LINE__); } } @@ -365,204 +353,148 @@ void ParameterHandlerBase::EnableSpecialProposal(const YAML::Node& param, const // ******************************************** // Set the covariance matrix for this class void ParameterHandlerBase::SetCovMatrix(TMatrixDSym *cov) { -// ******************************************** + // ******************************************** if (cov == nullptr) { - MACH3LOG_ERROR("Could not find covariance matrix you provided to {}", __func__ ); - throw MaCh3Exception(__FILE__ , __LINE__ ); - } - covMatrix = cov; - - invCovMatrix = static_cast(cov->Clone()); - invCovMatrix->Invert(); - //KS: ROOT has bad memory management, using standard double means we can decrease most operation by factor 2 simply due to cache hits - for (int i = 0; i < _fNumPar; i++) - { - for (int j = 0; j < _fNumPar; ++j) - { - InvertCovMatrix[i][j] = (*invCovMatrix)(i,j); - } + MACH3LOG_ERROR("Could not find covariance matrix you provided to {}", + __func__); + throw MaCh3Exception(__FILE__, __LINE__); } - SetThrowMatrix(cov); -} -// ******************************************** -void ParameterHandlerBase::ReserveMemory(const int SizeVec) { -// ******************************************** - _fNames = std::vector(SizeVec); - _fFancyNames = std::vector(SizeVec); - _fPreFitValue = std::vector(SizeVec); - _fError = std::vector(SizeVec); - _fCurrVal = std::vector(SizeVec); - _fPropVal = std::vector(SizeVec); - _fLowBound = std::vector(SizeVec); - _fUpBound = std::vector(SizeVec); - _fFlatPrior = std::vector(SizeVec); - _fIndivStepScale = std::vector(SizeVec); - _fSampleNames = std::vector>(_fNumPar); - - corr_throw = new double[SizeVec]; - // set random parameter vector (for correlated steps) - randParams = new double[SizeVec]; - - // Set the defaults to true - for(int i = 0; i < SizeVec; i++) { - _fPreFitValue.at(i) = 1.; - _fError.at(i) = 1.; - _fCurrVal.at(i) = 0.; - _fPropVal.at(i) = 0.; - _fLowBound.at(i) = -999.99; - _fUpBound.at(i) = 999.99; - _fFlatPrior.at(i) = false; - _fIndivStepScale.at(i) = 1.; - corr_throw[i] = 0.0; - randParams[i] = 0.0; + if (cov->GetNrows() != params.prefit.size()) { + MACH3LOG_ERROR( + "Passed covariance matrix size {0}x{0}, but have {1} parameters.", + ov->GetNrows(), params.prefit.size()); + throw MaCh3Exception(__FILE__, __LINE__); } - _fGlobalStepScale = 1.0; + params.covariance = M3::ROOTToEigen(*cov); + params.inv_covarance = params.covariance.invert(); + + SetThrowMatrix(cov); } // ******************************************** // Set all the covariance matrix parameters to a user-defined value // Might want to split this -void ParameterHandlerBase::SetPar(int i , double val) { -// ******************************************** - MACH3LOG_INFO("Over-riding {}: ", GetParName(i)); - MACH3LOG_INFO("_fPropVal ({}), _fCurrVal ({}), _fPreFitValue ({}) to ({})", _fPropVal[i], _fCurrVal[i], _fPreFitValue[i], val); - - _fPropVal[i] = val; - _fCurrVal[i] = val; - _fPreFitValue[i] = val; - - // Transfer the parameter values to the PCA basis - if (pca) PCAObj->TransferToPCA(); -} - -// ******************************************** -std::vector ParameterHandlerBase::GetProposed() const { -// ******************************************** - std::vector props(_fNumPar); - for (int i = 0; i < _fNumPar; ++i) props[i] = _fPropVal[i]; - return props; +void ParameterHandlerBase::SetPar(int i, double val) { + // ******************************************** + MACH3LOG_INFO("Overriding {}: ", GetParName(i)); + MACH3LOG_INFO( + "steps.proposed ({}), steps.current ({}), params.prefit ({}) to ({})", + steps.proposed[i], steps.current[i], params.prefit[i], val); + + steps.proposed[i] = val; + steps.current[i] = val; + params.prefit[i] = val; } // ************************************* // Throw the parameters according to the covariance matrix // This shouldn't be used in MCMC code ase it can break Detailed Balance; void ParameterHandlerBase::ThrowParameters() { -// ************************************* - // First draw new randParams - Randomize(); + // ************************************* - M3::MatrixVectorMulti(corr_throw, throwMatrixCholDecomp, randParams, _fNumPar); + // First draw a new random_vector + Randomize(); - // KS: We use PCA very rarely on top PCA functionality isn't implemented for this function. - // Use __builtin_expect to give compiler a hint which option is more likely, which should help - // with better optimisation. This isn't critical but more to have example - if (__builtin_expect(!pca, 1)) { - #ifdef MULTITHREAD - #pragma omp parallel for - #endif - for (int i = 0; i < _fNumPar; ++i) { - // Check if parameter is fixed first: if so don't randomly throw - if (IsParameterFixed(i)) continue; - - _fPropVal[i] = _fPreFitValue[i] + corr_throw[i]; - - int throws = 0; - // Try again if we the initial parameter proposal falls outside of the range of the parameter - while (_fPropVal[i] > _fUpBound[i] || _fPropVal[i] < _fLowBound[i]) { - randParams[i] = random_number[M3::GetThreadIndex()]->Gaus(0, 1); - const double corr_throw_single = M3::MatrixVectorMultiSingle(throwMatrixCholDecomp, randParams, _fNumPar, i); - _fPropVal[i] = _fPreFitValue[i] + corr_throw_single; - if (throws > 10000) - { - //KS: Since we are multithreading there is danger that those messages - //will be all over the place, small price to pay for faster code - MACH3LOG_WARN("Tried {} times to throw parameter {} but failed", throws, i); - MACH3LOG_WARN("Matrix: {}", matrixName); - MACH3LOG_WARN("Param: {}", _fNames[i]); - MACH3LOG_WARN("Setting _fPropVal: {} to {}", _fPropVal[i], _fPreFitValue[i]); - MACH3LOG_WARN("I live at {}:{}", __FILE__, __LINE__); - _fPropVal[i] = _fPreFitValue[i]; - //throw MaCh3Exception(__FILE__ , __LINE__ ); - } - throws++; + Eigen::VectorXd deltas = steps.l_proposal * throws.random_vector; + + for (int i = 0; i < throws.values.size(); ++i) { + + int tries = 0; + while (((params.prefit[i] + deltas[i]) < params.lowbound) || + ((params.prefit[i] + deltas[i]) >= params.upbound)) { + deltas[i] = steps.l_proposal.col(i) * rng.gaus(rng.e1); + tries++; + if (tries > 10000) { + // KS: Since we are multithreading there is danger that those messages + // will be all over the place, small price to pay for faster code + MACH3LOG_WARN("Tried {} times to throw parameter {} but failed", tries, + i); + MACH3LOG_WARN("Matrix: {}", settings.name); + MACH3LOG_WARN("Param: {}", params.name[i]); + MACH3LOG_WARN("Setting _fPropVal: {} to {}", steps.proposed[i], + params.prefit[i]); + MACH3LOG_WARN("I live at {}:{}", __FILE__, __LINE__); + deltas[i] = 0; } - _fCurrVal[i] = _fPropVal[i]; } + + steps.proposed = params.prefit + deltas; + steps.current = steps.proposed; } - else - { - PCAObj->ThrowParameters(random_number, throwMatrixCholDecomp, - randParams, corr_throw, - _fPreFitValue, _fLowBound, _fUpBound, _fNumPar); - } // end if pca - - // KS: At the end once we are happy with proposal do special proposal - SpecialStepProposal(); } // ************************************* // Throw each parameter within their 1 sigma range // Used to start the chain in different states void ParameterHandlerBase::RandomConfiguration() { -// ************************************* + // ************************************* // Have the 1 sigma for each parameter in each covariance class, sweet! - // Don't want to change the prior array because that's what determines our likelihood - // Want to change the _fPropVal, _fCurrVal, _fPreFitValue + // Don't want to change the prior array because that's what determines our + // likelihood Want to change the _fPropVal, _fCurrVal, _fPreFitValue // _fPreFitValue and the others will already be set for (int i = 0; i < _fNumPar; ++i) { // Check if parameter is fixed first: if so don't randomly throw - if (IsParameterFixed(i)) continue; + if (IsParameterFixed(i)) + continue; // Check that the sigma range is larger than the parameter range // If not, throw in the valid parameter range instead const double paramrange = _fUpBound[i] - _fLowBound[i]; - const double sigma = sqrt((*covMatrix)(i,i)); + const double sigma = sqrt((*covMatrix)(i, i)); double throwrange = sigma; - if (paramrange < sigma) throwrange = paramrange; + if (paramrange < sigma) + throwrange = paramrange; - _fPropVal[i] = _fPreFitValue[i] + random_number[0]->Gaus(0, 1)*throwrange; - // Try again if we the initial parameter proposal falls outside of the range of the parameter + _fPropVal[i] = _fPreFitValue[i] + random_number[0]->Gaus(0, 1) * throwrange; + // Try again if we the initial parameter proposal falls outside of the range + // of the parameter int throws = 0; while (_fPropVal[i] > _fUpBound[i] || _fPropVal[i] < _fLowBound[i]) { if (throws > 1000) { - MACH3LOG_WARN("Tried {} times to throw parameter {} but failed", throws, i); + MACH3LOG_WARN("Tried {} times to throw parameter {} but failed", throws, + i); MACH3LOG_WARN("Matrix: {}", matrixName); MACH3LOG_WARN("Param: {}", _fNames[i]); - throw MaCh3Exception(__FILE__ , __LINE__ ); + throw MaCh3Exception(__FILE__, __LINE__); } - _fPropVal[i] = _fPreFitValue[i] + random_number[0]->Gaus(0, 1)*throwrange; + _fPropVal[i] = + _fPreFitValue[i] + random_number[0]->Gaus(0, 1) * throwrange; throws++; } - MACH3LOG_INFO("Setting current step in {} param {} = {} from {}", matrixName, i, _fPropVal[i], _fCurrVal[i]); + MACH3LOG_INFO("Setting current step in {} param {} = {} from {}", + matrixName, i, _fPropVal[i], _fCurrVal[i]); _fCurrVal[i] = _fPropVal[i]; } - if (pca) PCAObj->TransferToPCA(); + if (pca) + PCAObj.TransferToPCA(); } // ************************************* // Set a single parameter -void ParameterHandlerBase::SetSingleParameter(const int parNo, const double parVal) { -// ************************************* - _fPropVal[parNo] = parVal; - _fCurrVal[parNo] = parVal; - MACH3LOG_DEBUG("Setting {} (parameter {}) to {})", GetParName(parNo), parNo, parVal); - if (pca) PCAObj->TransferToPCA(); +void ParameterHandlerBase::SetSingleParameter(const int parNo, + const double parVal) { + // ************************************* + steps.current[parNo] = parVal; + steps.proposed[parNo] = parVal; + MACH3LOG_DEBUG("Setting {} (parameter {}) to {})", GetParName(parNo), parNo, + parVal); + if (pca) + PCAObj.TransferToPCA(); } // ******************************************** -void ParameterHandlerBase::SetParCurrProp(const int parNo, const double parVal) { -// ******************************************** - _fPropVal[parNo] = parVal; - _fCurrVal[parNo] = parVal; - MACH3LOG_DEBUG("Setting {} (parameter {}) to {})", GetParName(parNo), parNo, parVal); - if (pca) PCAObj->TransferToPCA(); +void ParameterHandlerBase::SetParCurrProp(const int parNo, + const double parVal) { + // ******************************************** + SetSingleParameter(parNo, parVal); } // ************************************************ -// Propose a step for the set of systematics parameters this covariance class holds +// Propose a step for the set of systematics parameters this covariance class +// holds void ParameterHandlerBase::ProposeStep() { -// ************************************************ + // ************************************************ // Make the random numbers for the step proposal Randomize(); CorrelateSteps(); @@ -570,28 +502,32 @@ void ParameterHandlerBase::ProposeStep() { // KS: According to Dr Wallace we update using previous not proposed step // this way we do special proposal after adaptive after. // This way we can shortcut and skip rest of proposal - if(!doSpecialStepProposal) return; + if (!doSpecialStepProposal) + return; SpecialStepProposal(); } // ************************************************ void ParameterHandlerBase::SpecialStepProposal() { -// ************************************************ + // ************************************************ /// @warning KS: Following Asher comment we do "Step->Circular Bounds->Flip" // HW It should now automatically set dcp to be with [-pi, pi] - for (size_t i = 0; i < CircularBoundsIndex.size(); ++i) { - const int index = CircularBoundsIndex[i]; - if(!IsParameterFixed(index)) - CircularParBounds(index, CircularBoundsValues[i].first, CircularBoundsValues[i].second); + for (size_t i = 0; i < special_proposal.CircularBoundsIndex.size(); ++i) { + const int index = special_proposal.CircularBoundsIndex[i]; + if (!IsParameterFixed(index)) + CircularParBounds(index, special_proposal.CircularBoundsValues[i].first, + special_proposal.CircularBoundsValues[i].second); } - // Okay now we've done the standard steps, we can add in our nice flips hierarchy flip first - for (size_t i = 0; i < FlipParameterIndex.size(); ++i) { - const int index = FlipParameterIndex[i]; - if(!IsParameterFixed(index)) - FlipParameterValue(FlipParameterIndex[i], FlipParameterPoint[i]); + // Okay now we've done the standard steps, we can add in our nice flips + // hierarchy flip first + for (size_t i = 0; i < special_proposal.FlipParameterIndex.size(); ++i) { + const int index = special_proposal.FlipParameterIndex[i]; + if (!IsParameterFixed(index)) + FlipParameterValue(special_proposal.FlipParameterIndex[i], + special_proposal.FlipParameterPoint[i]); } } @@ -600,97 +536,65 @@ void ParameterHandlerBase::SpecialStepProposal() { // Used the proposal kernel and the current parameter value to set proposed step // Also get a new random number for the randParams void ParameterHandlerBase::Randomize() _noexcept_ { -// ************************************************ - if (!pca) { - //KS: By multithreading here we gain at least factor 2 with 8 threads with ND only fit - #ifdef MULTITHREAD - #pragma omp parallel for - #endif - for (int i = 0; i < _fNumPar; ++i) { - // If parameter isn't fixed - if (!IsParameterFixed(i) > 0.0) { - randParams[i] = random_number[M3::GetThreadIndex()]->Gaus(0, 1); - // If parameter IS fixed - } else { - randParams[i] = 0.0; - } - } // end for - // If we're in the PCA basis we instead throw parameters there (only _fNumParPCA parameter) - } else { - // Scale the random parameters by the sqrt of eigen values for the throw - #ifdef MULTITHREAD - #pragma omp parallel for - #endif - for (int i = 0; i < PCAObj->GetNumberPCAedParameters(); ++i) - { - // If parameter IS fixed or out of bounds - if (PCAObj->IsParameterFixedPCA(i)) { - randParams[i] = 0.0; - } else { - randParams[i] = random_number[M3::GetThreadIndex()]->Gaus(0,1); - } - } - } + // ************************************************ + throws.random_vector = Eigen::VectorXd::NullaryExpr( + params.prefit.size(), [&](int) { return rng.gaus(rng.e1); }); } // ************************************************ -// Correlate the steps by setting the proposed step of a parameter to its current value + some correlated throw +// Correlate the steps by setting the proposed step of a parameter to its +// current value + some correlated throw void ParameterHandlerBase::CorrelateSteps() _noexcept_ { -// ************************************************ - //KS: Using custom function compared to ROOT one with 8 threads we have almost factor 2 performance increase, by replacing TMatrix with just double we increase it even more - M3::MatrixVectorMulti(corr_throw, throwMatrixCholDecomp, randParams, _fNumPar); + // ************************************************ - // If not doing PCA if (!pca) { - #ifdef MULTITHREAD - #pragma omp parallel for - #endif +#ifdef MULTITHREAD +#pragma omp parallel for +#endif for (int i = 0; i < _fNumPar; ++i) { if (!IsParameterFixed(i) > 0.) { - _fPropVal[i] = _fCurrVal[i] + corr_throw[i]*_fGlobalStepScale*_fIndivStepScale[i]; + _fPropVal[i] = _fCurrVal[i] + + corr_throw[i] * _fGlobalStepScale * _fIndivStepScale[i]; } } - // If doing PCA throw uncorrelated in PCA basis (orthogonal basis by definition) - } else { - PCAObj->CorrelateSteps(_fIndivStepScale, _fGlobalStepScale, randParams, corr_throw); + // If doing PCA throw uncorrelated in PCA basis (orthogonal basis by + // definition) + } else { + PCAObj.CorrelateSteps(_fIndivStepScale, _fGlobalStepScale, randParams, + corr_throw); } } // ******************************************** // Update so that current step becomes the previously proposed step void ParameterHandlerBase::AcceptStep() _noexcept_ { -// ******************************************** - if (!pca) { - #ifdef MULTITHREAD - #pragma omp parallel for - #endif - for (int i = 0; i < _fNumPar; ++i) { - // Update state so that current state is proposed state - _fCurrVal[i] = _fPropVal[i]; - } - } else { - PCAObj->AcceptStep(); - } + // ******************************************** + steps.current = steps.proposed; if (AdaptiveHandler) { - AdaptiveHandler->IncrementAcceptedSteps(); + AdaptiveHandler.IncrementAcceptedSteps(); } } // ************************************* -//HW: This method is a tad hacky but modular arithmetic gives me a headache. -void ParameterHandlerBase::CircularParBounds(const int index, const double LowBound, const double UpBound) { -// ************************************* - if(_fPropVal[index] > UpBound) { - _fPropVal[index] = LowBound + std::fmod(_fPropVal[index] - UpBound, UpBound - LowBound); +// HW: This method is a tad hacky but modular arithmetic gives me a headache. +void ParameterHandlerBase::CircularParBounds(const int index, + const double LowBound, + const double UpBound) { + // ************************************* + if (_fPropVal[index] > UpBound) { + _fPropVal[index] = + LowBound + std::fmod(_fPropVal[index] - UpBound, UpBound - LowBound); } else if (_fPropVal[index] < LowBound) { - _fPropVal[index] = UpBound - std::fmod(LowBound - _fPropVal[index], UpBound - LowBound); + _fPropVal[index] = + UpBound - std::fmod(LowBound - _fPropVal[index], UpBound - LowBound); } } // ************************************* -void ParameterHandlerBase::FlipParameterValue(const int index, const double FlipPoint) { -// ************************************* - if(random_number[0]->Uniform() < 0.5) { +void ParameterHandlerBase::FlipParameterValue(const int index, + const double FlipPoint) { + // ************************************* + if (random_number[0]->Uniform() < 0.5) { _fPropVal[index] = 2 * FlipPoint - _fPropVal[index]; } } @@ -699,46 +603,49 @@ void ParameterHandlerBase::FlipParameterValue(const int index, const double Flip // Throw the proposed parameter by mag sigma // Should really just have the user specify this throw by having argument double void ParameterHandlerBase::ThrowParProp(const double mag) { -// ******************************************** + // ******************************************** Randomize(); if (!pca) { // Make the correlated throw - M3::MatrixVectorMulti(corr_throw, throwMatrixCholDecomp, randParams, _fNumPar); + M3::MatrixVectorMulti(corr_throw, throwMatrixCholDecomp, randParams, + _fNumPar); // Number of sigmas we throw for (int i = 0; i < _fNumPar; i++) { if (!IsParameterFixed(i) > 0.) - _fPropVal[i] = _fCurrVal[i] + corr_throw[i]*mag; + _fPropVal[i] = _fCurrVal[i] + corr_throw[i] * mag; } } else { - PCAObj->ThrowParProp(mag, randParams); + PCAObj.ThrowParProp(mag, randParams); } } // ******************************************** // Helper function to throw the current parameter by mag sigmas // Can study bias in MCMC with this; put different starting parameters void ParameterHandlerBase::ThrowParCurr(const double mag) { -// ******************************************** + // ******************************************** Randomize(); if (!pca) { // Get the correlated throw vector - M3::MatrixVectorMulti(corr_throw, throwMatrixCholDecomp, randParams, _fNumPar); + M3::MatrixVectorMulti(corr_throw, throwMatrixCholDecomp, randParams, + _fNumPar); // The number of sigmas to throw - // Should probably have this as a default parameter input to the function instead + // Should probably have this as a default parameter input to the function + // instead for (int i = 0; i < _fNumPar; i++) { - if (!IsParameterFixed(i) > 0.){ - _fCurrVal[i] = corr_throw[i]*mag; + if (!IsParameterFixed(i) > 0.) { + _fCurrVal[i] = corr_throw[i] * mag; } } } else { - PCAObj->ThrowParCurr(mag, randParams); + PCAObj.ThrowParCurr(mag, randParams); } } // ******************************************** // Function to print the prior values void ParameterHandlerBase::PrintNominal() const { -// ******************************************** + // ******************************************** MACH3LOG_INFO("Prior values for {} ParameterHandler:", GetName()); - for (int i = 0; i < _fNumPar; i++) { + for (int i = 0; i < params.name.size(); i++) { MACH3LOG_INFO(" {} {} ", GetParFancyName(i), GetParInit(i)); } } @@ -746,578 +653,450 @@ void ParameterHandlerBase::PrintNominal() const { // ******************************************** // Function to print the prior, current and proposed values void ParameterHandlerBase::PrintNominalCurrProp() const { -// ******************************************** + // ******************************************** MACH3LOG_INFO("Printing parameters for {}", GetName()); - // Dump out the PCA parameters too - if (pca) { - PCAObj->Print(); - } - MACH3LOG_INFO("{:<30} {:<10} {:<10} {:<10}", "Name", "Prior", "Current", "Proposed"); - for (int i = 0; i < _fNumPar; ++i) { - MACH3LOG_INFO("{:<30} {:<10.2f} {:<10.2f} {:<10.2f}", GetParFancyName(i), _fPreFitValue[i], _fCurrVal[i], _fPropVal[i]); + MACH3LOG_INFO("{:<30} {:<10} {:<10} {:<10}", "Name", "Prior", "Current", + "Proposed"); + for (int i = 0; i < params.prefit.size(); ++i) { + MACH3LOG_INFO("{:<30} {:<10.2f} {:<10.2f} {:<10.2f}", GetParFancyName(i), + params.prefit[i], params.current[i], params.proposed[i]); } } // ******************************************** -// Get the likelihood in the case where we want to include priors on the parameters -// _fFlatPrior stores if we want to evaluate the likelihood for the given parameter +// Get the likelihood in the case where we want to include priors on the +// parameters _fFlatPrior stores if we want to evaluate the likelihood for the +// given parameter // true = don't evaluate likelihood (so run without a prior) // false = evaluate likelihood (so run with a prior) + double ParameterHandlerBase::CalcLikelihood() const _noexcept_ { -// ******************************************** - double logL = 0.0; - #ifdef MULTITHREAD - #pragma omp parallel for reduction(+:logL) - #endif - for(int i = 0; i < _fNumPar; ++i){ - if(_fFlatPrior[i]){ - //HW: Flat prior, no need to calculate anything - continue; - } - // KS: Precalculate Diff once per "i" without doing this for every "j" - const double Diff = _fPropVal[i] - _fPreFitValue[i]; - #ifdef MULTITHREAD - #pragma omp simd - #endif - for (int j = 0; j <= i; ++j) { - if (!_fFlatPrior[j]) { - //KS: Since matrix is symmetric we can calculate non diagonal elements only once and multiply by 2, can bring up to factor speed decrease. - double scale = (i != j) ? 1. : 0.5; - logL += scale * Diff * (_fPropVal[j] - _fPreFitValue[j])*InvertCovMatrix[i][j]; - } - } - } - return logL; + // ******************************************** + // filter out parameters with flat priors from the pull + auto diff = + params.flatprior.select(Eigen::VectorXd::Zeros(params.prefit.size()), + params.proposed - params.prefit); + return diff.T * params.inv_covariance * diff; } // ******************************************** int ParameterHandlerBase::CheckBounds() const _noexcept_ { -// ******************************************** - int NOutside = 0; - #ifdef MULTITHREAD - #pragma omp parallel for reduction(+:NOutside) - #endif - for (int i = 0; i < _fNumPar; ++i){ - if(_fPropVal[i] > _fUpBound[i] || _fPropVal[i] < _fLowBound[i]){ - NOutside++; - } - } - return NOutside; + // ******************************************** + return ((params.proposed.array() < params.lowbound.array()) || + (params.proposed.array() >= params.upbound.array())) + .count(); } // ******************************************** double ParameterHandlerBase::GetLikelihood() { -// ******************************************** + // ******************************************** // Default behaviour is to reject negative values + do std llh calculation const int NOutside = CheckBounds(); - - if(NOutside > 0) return NOutside*M3::_LARGE_LOGL_; + + if (NOutside > 0) + return NOutside * M3::_LARGE_LOGL_; return CalcLikelihood(); } // ******************************************** // Sets the proposed parameters to the prior values -void ParameterHandlerBase::SetParameters(const std::vector& pars) { -// ******************************************** +void ParameterHandlerBase::SetParameters(const std::vector &pars) { + // ******************************************** // If empty, set the proposed to prior if (pars.empty()) { - // For xsec this means setting to the prior (because prior is the prior) - for (int i = 0; i < _fNumPar; i++) { - _fPropVal[i] = _fPreFitValue[i]; - } + steps.proposed = params.prefit; // If not empty, set the parameters to the specified } else { if (pars.size() != size_t(_fNumPar)) { - MACH3LOG_ERROR("Parameter arrays of incompatible size! Not changing parameters! {} has size {} but was expecting {}", matrixName, pars.size(), _fNumPar); - throw MaCh3Exception(__FILE__ , __LINE__ ); + MACH3LOG_ERROR("Parameter arrays of incompatible size! Not changing " + "parameters! {} has size {} but was expecting {}", + matrixName, pars.size(), _fNumPar); + throw MaCh3Exception(__FILE__, __LINE__); } - int parsSize = int(pars.size()); - for (int i = 0; i < parsSize; i++) { - //Make sure that you are actually passing a number to set the parameter to - if(std::isnan(pars[i])) { - MACH3LOG_ERROR("Trying to set parameter value to a nan for parameter {} in matrix {}. This will not go well!", GetParName(i), matrixName); - throw MaCh3Exception(__FILE__ , __LINE__ ); + for (int i = 0; i < pars.size(); i++) { + // Make sure that you are actually passing a number to set the parameter + // to + if (std::isnan(pars[i])) { + MACH3LOG_ERROR("Trying to set parameter value to a nan for parameter " + "{} in matrix {}. This will not go well!", + GetParName(i), matrixName); + throw MaCh3Exception(__FILE__, __LINE__); } else { - _fPropVal[i] = pars[i]; + steps.proposed[i] = pars[i]; } } } - // And if pca make the transfer - if (pca) { - PCAObj->TransferToPCA(); - PCAObj->TransferToParam(); - } } // ******************************************** void ParameterHandlerBase::SetBranches(TTree &tree, bool SaveProposal) { -// ******************************************** + // ******************************************** // loop over parameters and set a branch - for (int i = 0; i < _fNumPar; ++i) { - tree.Branch(_fNames[i].c_str(), &_fCurrVal[i], Form("%s/D", _fNames[i].c_str())); + for (int i = 0; i < params.name.size(); ++i) { + tree.Branch(params.name[i].c_str(), params.current.data() + i, + Form("%s/D", params.name[i].c_str())); } // When running PCA, also save PCA parameters if (pca) { - PCAObj->SetBranches(tree, SaveProposal, _fNames); + PCAObj.SetBranches(tree, SaveProposal, _fNames); } - if(SaveProposal) - { + if (SaveProposal) { // loop over parameters and set a branch - for (int i = 0; i < _fNumPar; ++i) { - tree.Branch(Form("%s_Prop", _fNames[i].c_str()), &_fPropVal[i], Form("%s_Prop/D", _fNames[i].c_str())); + for (int i = 0; i < params.name.size(); ++i) { + tree.Branch(Form("%s_Prop", params.name[i].c_str()), + params.proposed.data() + i, + Form("%s_Prop/D", params.name[i].c_str())); } } - if(use_adaptive && AdaptiveHandler->GetUseRobbinsMonro()){ - tree.Branch(Form("GlobalStepScale_%s", GetName().c_str()), &_fGlobalStepScale, Form("GlobalStepScale_%s/D", GetName().c_str())); + if (use_adaptive && AdaptiveHandler.GetUseRobbinsMonro()) { + tree.Branch(Form("GlobalStepScale_%s", GetName().c_str()), + &_fGlobalStepScale, + Form("GlobalStepScale_%s/D", GetName().c_str())); } } // ******************************************** -void ParameterHandlerBase::SetStepScale(const double scale, const bool verbose) { -// ******************************************** - if(scale <= 0) { - MACH3LOG_ERROR("You are trying so set StepScale to 0 or negative this will not work"); - throw MaCh3Exception(__FILE__ , __LINE__ ); +void ParameterHandlerBase::SetStepScale(const double scale, + const bool verbose) { + // ******************************************** + if (scale <= 0) { + MACH3LOG_ERROR("Invalid step scale {}", scale); + throw MaCh3Exception(__FILE__, __LINE__); } - if(verbose){ + if (verbose) { MACH3LOG_INFO("{} setStepScale() = {}", GetName(), scale); - const double SuggestedScale = 2.38/std::sqrt(_fNumPar); - if(std::fabs(scale - SuggestedScale)/SuggestedScale > 1) { - MACH3LOG_WARN("Defined Global StepScale is {}, while suggested suggested {}", scale, SuggestedScale); + const double SuggestedScale = 2.38 / std::sqrt(_fNumPar); + if ((std::fabs(scale - SuggestedScale) / SuggestedScale) > 1) { + MACH3LOG_WARN( + "Defined Global StepScale is {}, while suggested suggested {}", scale, + SuggestedScale); } } - _fGlobalStepScale = scale; + steps.global_scale = scale; } // ******************************************** -int ParameterHandlerBase::GetParIndex(const std::string& name) const { -// ******************************************** - int Index = M3::_BAD_INT_; - for (int i = 0; i <_fNumPar; ++i) { - if(name == _fFancyNames[i]) { - Index = i; - break; - } +int ParameterHandlerBase::GetParIndex(const std::string &name) const { + // ******************************************** + auto idx = std::find(params.name.begin(), params.name.end(), name); + if (idx == params.name.end()) { + MACH3LOG_ERROR("No parameter with the name {} exists.", name); + throw MaCh3Exception(__FILE__, __LINE__); } - return Index; + return std::distance(params.name.begin(), idx); } // ******************************************** void ParameterHandlerBase::SetFixAllParameters() { -// ******************************************** - // Check if the parameter is fixed and if not, toggle fix it - for (int i = 0; i < _fNumPar; ++i) - if(!IsParameterFixed(i)) ToggleFixParameter(i); + // ******************************************** + params.fixed = Eigen::VectorXi::Ones(params.prefit.size()); } // ******************************************** void ParameterHandlerBase::SetFixParameter(const int i) { -// ******************************************** - // Check if the parameter is fixed and if not, toggle fix it - if(!IsParameterFixed(i)) ToggleFixParameter(i); + // ******************************************** + params.fixed[i] = 1; } // ******************************************** -void ParameterHandlerBase::SetFixParameter(const std::string& name) { -// ******************************************** - // Check if the parameter is fixed and if not, toggle fix it - if(!IsParameterFixed(name)) ToggleFixParameter(name); +void ParameterHandlerBase::SetFixParameter(const std::string &name) { + // ******************************************** + SetFixParameter(GetParIndex(name)); } // ******************************************** void ParameterHandlerBase::SetFreeAllParameters() { -// ******************************************** - // Check if the parameter is fixed and if not, toggle fix it - for (int i = 0; i < _fNumPar; ++i) - if(IsParameterFixed(i)) ToggleFixParameter(i); + // ******************************************** + params.fixed = Eigen::VectorXi::Zeros(params.prefit.size()); } // ******************************************** void ParameterHandlerBase::SetFreeParameter(const int i) { -// ******************************************** - // Check if the parameter is fixed and if not, toggle fix it - if(IsParameterFixed(i)) ToggleFixParameter(i); + // ******************************************** + params.fixed[i] = 0; } // ******************************************** -void ParameterHandlerBase::SetFreeParameter(const std::string& name) { -// ******************************************** - // Check if the parameter is fixed and if not, toggle fix it - if(IsParameterFixed(name)) ToggleFixParameter(name); +void ParameterHandlerBase::SetFreeParameter(const std::string &name) { + // ******************************************** + SetFreeParameter(GetParIndex(name)); } // ******************************************** void ParameterHandlerBase::ToggleFixAllParameters() { -// ******************************************** - // toggle fix/free all parameters - if(!pca) for (int i = 0; i < _fNumPar; i++) ToggleFixParameter(i); - else PCAObj->ToggleFixAllParameters(_fNames); + // ******************************************** + throw std::runtime_error("Don't Toggle parameters"); } // ******************************************** void ParameterHandlerBase::ToggleFixParameter(const int i) { -// ******************************************** - if(!pca) { - if (i > _fNumPar) { - MACH3LOG_ERROR("Can't {} for parameter {} because size of covariance ={}", __func__, i, _fNumPar); - MACH3LOG_ERROR("Fix this in your config file please!"); - throw MaCh3Exception(__FILE__ , __LINE__ ); - } else { - _fError[i] *= -1.0; - if(IsParameterFixed(i)) MACH3LOG_INFO("Setting {}(parameter {}) to fixed at {}", GetParFancyName(i), i, _fCurrVal[i]); - else MACH3LOG_INFO("Setting {}(parameter {}) free", GetParFancyName(i), i); - } - if( (_fCurrVal[i] > _fUpBound[i] || _fCurrVal[i] < _fLowBound[i]) && IsParameterFixed(i) ) { - MACH3LOG_ERROR("Parameter {} (index {}) is fixed at {}, which is outside of its bounds [{}, {}]", GetParFancyName(i), i, _fCurrVal[i], _fLowBound[i], _fUpBound[i]); - throw MaCh3Exception(__FILE__ , __LINE__ ); - } - } else { - PCAObj->ToggleFixParameter(i, _fNames); - } + // ******************************************** + throw std::runtime_error("Don't Toggle parameters"); } // ******************************************** -void ParameterHandlerBase::ToggleFixParameter(const std::string& name) { -// ******************************************** - const int Index = GetParIndex(name); - if(Index != M3::_BAD_INT_) { - ToggleFixParameter(Index); - return; - } - - MACH3LOG_WARN("I couldn't find parameter with name {}, therefore will not fix it", name); +void ParameterHandlerBase::ToggleFixParameter(const std::string &name) { + // ******************************************** + throw std::runtime_error("Don't Toggle parameters"); } // ******************************************** -bool ParameterHandlerBase::IsParameterFixed(const std::string& name) const { -// ******************************************** - const int Index = GetParIndex(name); - if(Index != M3::_BAD_INT_) { - return IsParameterFixed(Index); - } - - MACH3LOG_WARN("I couldn't find parameter with name {}, therefore don't know if it fixed", name); - return false; +bool ParameterHandlerBase::IsParameterFixed(const std::string &name) const { + // ******************************************** + return IsParameterFixed(GetParIndex(name)); } // ******************************************** void ParameterHandlerBase::SetFlatPrior(const int i, const bool eL) { -// ******************************************** - if (i > _fNumPar) { - MACH3LOG_INFO("Can't {} for Cov={}/Param={} because size of Covariance = {}", __func__, GetName(), i, _fNumPar); + // ******************************************** + if (i > params.flatprior.size()) { + MACH3LOG_INFO( + "Can't {} for Cov={}/Param={} because size of Covariance = {}", + __func__, GetName(), i, params.flatprior.size()); MACH3LOG_ERROR("Fix this in your config file please!"); - throw MaCh3Exception(__FILE__ , __LINE__ ); + throw MaCh3Exception(__FILE__, __LINE__); } else { - if(eL){ - MACH3LOG_INFO("Setting {} (parameter {}) to flat prior", GetParName(i), i); - } - else{ + if (eL) { + MACH3LOG_INFO("Setting {} (parameter {}) to flat prior", GetParName(i), + i); + } else { // HW :: This is useful - MACH3LOG_INFO("Setting {} (parameter {}) to non-flat prior", GetParName(i), i); + MACH3LOG_INFO("Setting {} (parameter {}) to non-flat prior", + GetParName(i), i); } - _fFlatPrior[i] = eL; + params.flatprior[i] = eL; } } // ******************************************** -void ParameterHandlerBase::SetIndivStepScale(const std::vector& stepscale) { -// ******************************************** - if (int(stepscale.size()) != _fNumPar) - { - MACH3LOG_WARN("Stepscale vector not equal to number of parameters. Quitting.."); +void ParameterHandlerBase::SetIndivStepScale( + const std::vector &stepscale) { + // ******************************************** + if (int(stepscale.size()) != params.stepscale.size()) { + MACH3LOG_WARN( + "Stepscale vector not equal to number of parameters. Quitting.."); MACH3LOG_WARN("Size of argument vector: {}", stepscale.size()); - MACH3LOG_WARN("Expected size: {}", _fNumPar); + MACH3LOG_WARN("Expected size: {}", params.stepscale.size()); return; } - for (int iParam = 0 ; iParam < _fNumPar; iParam++) { - _fIndivStepScale[iParam] = stepscale[iParam]; - } + params.stepscale = M3::StdVectorToEigen(stepscale); PrintIndivStepScale(); } // ******************************************** void ParameterHandlerBase::PrintIndivStepScale() const { -// ******************************************** + // ******************************************** MACH3LOG_INFO("============================================================"); MACH3LOG_INFO("{:<{}} | {:<11}", "Parameter:", PrintLength, "Step scale:"); - for (int iParam = 0; iParam < _fNumPar; iParam++) { - MACH3LOG_INFO("{:<{}} | {:<11}", _fFancyNames[iParam].c_str(), PrintLength, _fIndivStepScale[iParam]); + for (int iParam = 0; iParam < params.fancyname.size(); iParam++) { + MACH3LOG_INFO("{:<{}} | {:<11}", params.fancyname[iParam].c_str(), + PrintLength, params.stepscale[iParam]); } MACH3LOG_INFO("============================================================"); } -// ******************************************** -//Makes sure that matrix is positive-definite by adding a small number to on-diagonal elements -void ParameterHandlerBase::MakePosDef(TMatrixDSym *cov) { -// ******************************************** - if(cov == nullptr){ - cov = &*covMatrix; - MACH3LOG_WARN("Passed nullptr to cov matrix in {}", matrixName); - } - - M3::MakeMatrixPosDef(cov); -} - // ******************************************** void ParameterHandlerBase::ResetIndivStepScale() { -// ******************************************** - std::vector stepScales(_fNumPar); - for (int i = 0; i <_fNumPar; i++) { - stepScales[i] = 1.; - } - _fGlobalStepScale = 1.0; - SetIndivStepScale(stepScales); + // ******************************************** + steps.global_scale = 1.0; + SetIndivStepScale(std::vector(1, params.stepscale.size())); } // ******************************************** -// HW: Code for throwing from separate throw matrix, needs to be set after init to ensure pos-def -void ParameterHandlerBase::SetThrowMatrix(TMatrixDSym *cov){ -// ******************************************** +// HW: Code for throwing from separate throw matrix, needs to be set after init +// to ensure pos-def +void ParameterHandlerBase::SetThrowMatrix(TMatrixDSym *cov) { + // ******************************************** - if (cov == nullptr) { - MACH3LOG_ERROR("Could not find covariance matrix you provided to {}", __func__); - throw MaCh3Exception(__FILE__ , __LINE__ ); + if (cov == nullptr) { + MACH3LOG_ERROR("Could not find covariance matrix you provided to {}", + __func__); + throw MaCh3Exception(__FILE__, __LINE__); } - if (covMatrix->GetNrows() != cov->GetNrows()) { - MACH3LOG_ERROR("Matrix given for throw Matrix is not the same size as the covariance matrix stored in object!"); - MACH3LOG_ERROR("Stored covariance matrix size: {}", covMatrix->GetNrows()); + if (params.l_proposal.rows() != cov->GetNrows()) { + MACH3LOG_ERROR("Matrix given for throw Matrix is not the same size as the " + "proposal matrix stored in object!"); + MACH3LOG_ERROR("Stored proposal matrix size: {}", params.l_proposal.rows()); MACH3LOG_ERROR("Given matrix size: {}", cov->GetNrows()); - throw MaCh3Exception(__FILE__ , __LINE__ ); + throw MaCh3Exception(__FILE__, __LINE__); } - throwMatrix = static_cast(cov->Clone()); - if(use_adaptive && AdaptiveHandler->AdaptionUpdate()) MakeClosestPosDef(throwMatrix); - else MakePosDef(throwMatrix); - - auto throwMatrix_CholDecomp = M3::GetCholeskyDecomposedMatrix(*throwMatrix, matrixName); - - //KS: ROOT has bad memory management, using standard double means we can decrease most operation by factor 2 simply due to cache hits - #ifdef MULTITHREAD - #pragma omp parallel for collapse(2) - #endif - for (int i = 0; i < _fNumPar; ++i) - { - for (int j = 0; j < _fNumPar; ++j) - { - throwMatrixCholDecomp[i][j] = throwMatrix_CholDecomp[i][j]; - } + if (use_adaptive && AdaptiveHandler.AdaptionUpdate()) { + M3::MakeClosestPosDef(cov); + params.l_proposal = Eigen::LLT(M3::ROOTToEigen(cov)).L(); + } else { + params.l_proposal = + Eigen::LLT(M3::MakePosDef(M3::ROOTToEigen(*cov))).L(); } } // ******************************************** -void ParameterHandlerBase::UpdateThrowMatrix(TMatrixDSym *cov){ -// ******************************************** - delete throwMatrix; - throwMatrix = nullptr; +void ParameterHandlerBase::UpdateThrowMatrix(TMatrixDSym *cov) { + // ******************************************** SetThrowMatrix(cov); } // ******************************************** // HW : Here be adaption -void ParameterHandlerBase::InitialiseAdaption(const YAML::Node& adapt_manager){ -// ******************************************** - if(PCAObj){ - MACH3LOG_ERROR("PCA has been enabled and now trying to enable Adaption. Right now both configuration don't work with each other"); - throw MaCh3Exception(__FILE__ , __LINE__ ); +void ParameterHandlerBase::InitialiseAdaption(const YAML::Node &adapt_manager) { + // ******************************************** + if (PCAObj) { + MACH3LOG_ERROR("PCA has been enabled and now trying to enable Adaption. " + "Right now both configuration don't work with each other"); + throw MaCh3Exception(__FILE__, __LINE__); + } + if (AdaptiveHandler) { + MACH3LOG_ERROR("Adaptive Handler has already been initialise can't do it " + "again so skipping."); + return; } - if(AdaptiveHandler){ - MACH3LOG_ERROR("Adaptive Handler has already been initialise can't do it again so skipping."); + + // Now we read the general settings [these SHOULD be common across all + // matrices!] + bool success = AdaptiveHandler.InitFromConfig(adapt_manager, matrixName, + &_fCurrVal, &_fError); + if (!success) { return; } - AdaptiveHandler = std::make_unique(); - // Now we read the general settings [these SHOULD be common across all matrices!] - bool success = AdaptiveHandler->InitFromConfig(adapt_manager, matrixName, &_fCurrVal, &_fError); - if(!success) return; - AdaptiveHandler->Print(); + AdaptiveHandler.Print(); // Next let"s check for external matrices // We"re going to grab this info from the YAML manager - if(!GetFromManager(adapt_manager["AdaptionOptions"]["Covariance"][matrixName]["UseExternalMatrix"], false, __FILE__ , __LINE__)) { - MACH3LOG_WARN("Not using external matrix for {}, initialising adaption from scratch", matrixName); - // If we don't have a covariance matrix to start from for adaptive tune we need to make one! + if (!GetFromManager(adapt_manager["AdaptionOptions"]["Covariance"] + [matrixName]["UseExternalMatrix"], + false, __FILE__, __LINE__)) { + MACH3LOG_WARN( + "Not using external matrix for {}, initialising adaption from scratch", + matrixName); + // If we don't have a covariance matrix to start from for adaptive tune we + // need to make one! use_adaptive = true; - AdaptiveHandler->CheckMatrixValidityForAdaption(GetCovMatrix()); - AdaptiveHandler->CreateNewAdaptiveCovariance(); + AdaptiveHandler.CheckMatrixValidityForAdaption(GetCovMatrix()); + AdaptiveHandler.CreateNewAdaptiveCovariance(); return; } // Finally, we accept that we want to read the matrix from a file! - auto external_file_name = GetFromManager(adapt_manager["AdaptionOptions"]["Covariance"][matrixName]["ExternalMatrixFileName"], "", __FILE__ , __LINE__); - auto external_matrix_name = GetFromManager(adapt_manager["AdaptionOptions"]["Covariance"][matrixName]["ExternalMatrixName"], "", __FILE__ , __LINE__); - auto external_mean_name = GetFromManager(adapt_manager["AdaptionOptions"]["Covariance"][matrixName]["ExternalMeansName"], "", __FILE__ , __LINE__); - - AdaptiveHandler->SetThrowMatrixFromFile(external_file_name, external_matrix_name, external_mean_name, use_adaptive); - SetThrowMatrix(AdaptiveHandler->GetAdaptiveCovariance()); + auto external_file_name = GetFromManager( + adapt_manager["AdaptionOptions"]["Covariance"][matrixName] + ["ExternalMatrixFileName"], + "", __FILE__, __LINE__); + auto external_matrix_name = GetFromManager( + adapt_manager["AdaptionOptions"]["Covariance"][matrixName] + ["ExternalMatrixName"], + "", __FILE__, __LINE__); + auto external_mean_name = GetFromManager( + adapt_manager["AdaptionOptions"]["Covariance"][matrixName] + ["ExternalMeansName"], + "", __FILE__, __LINE__); + + AdaptiveHandler.SetThrowMatrixFromFile(external_file_name, + external_matrix_name, + external_mean_name, use_adaptive); + SetThrowMatrix(AdaptiveHandler.GetAdaptiveCovariance()); ResetIndivStepScale(); - MACH3LOG_INFO("Successfully Set External Throw Matrix Stored in {}", external_file_name); + MACH3LOG_INFO("Successfully Set External Throw Matrix Stored in {}", + external_file_name); } // ******************************************** // Truely adaptive MCMC! -void ParameterHandlerBase::UpdateAdaptiveCovariance(){ -// ******************************************** +void ParameterHandlerBase::UpdateAdaptiveCovariance() { + // ******************************************** // Updates adaptive matrix // First we update the total means // Skip this if we're at a large number of steps - if(AdaptiveHandler->SkipAdaption()) { - AdaptiveHandler->IncrementNSteps(); + if (AdaptiveHandler.SkipAdaption()) { + AdaptiveHandler.IncrementNSteps(); return; } /// Need to adjust the scale every step - if(AdaptiveHandler->GetUseRobbinsMonro()){ - bool verbose=false; - #ifdef DEBUG - verbose=true; - #endif - AdaptiveHandler->UpdateRobbinsMonroScale(); - SetStepScale(AdaptiveHandler->GetAdaptionScale(), verbose); + if (AdaptiveHandler.GetUseRobbinsMonro()) { + bool verbose = false; +#ifdef DEBUG + verbose = true; +#endif + AdaptiveHandler.UpdateRobbinsMonroScale(); + SetStepScale(AdaptiveHandler.GetAdaptionScale(), verbose); } // Call main adaption function - AdaptiveHandler->UpdateAdaptiveCovariance(); + AdaptiveHandler.UpdateAdaptiveCovariance(); // Set scales to 1 * optimal scale - if(AdaptiveHandler->IndivStepScaleAdapt()) { + if (AdaptiveHandler.IndivStepScaleAdapt()) { ResetIndivStepScale(); - SetStepScale(AdaptiveHandler->GetAdaptionScale()); + SetStepScale(AdaptiveHandler.GetAdaptionScale()); } - if(AdaptiveHandler->UpdateMatrixAdapt()) { - TMatrixDSym* update_matrix = static_cast(AdaptiveHandler->GetAdaptiveCovariance()->Clone()); - UpdateThrowMatrix(update_matrix); //Now we update and continue! - //Also Save the adaptive to file - AdaptiveHandler->SaveAdaptiveToFile(AdaptiveHandler->GetOutFileName(), GetName()); + if (AdaptiveHandler.UpdateMatrixAdapt()) { + UpdateThrowMatrix( + AdaptiveHandler.GetAdaptiveCovariance()); // Now we update and continue! + // Also Save the adaptive to file + AdaptiveHandler.SaveAdaptiveToFile(AdaptiveHandler.GetOutFileName(), + GetName()); } - AdaptiveHandler->IncrementNSteps(); + AdaptiveHandler.IncrementNSteps(); } // ******************************************** -//HW: Finds closest possible positive definite matrix in Frobenius Norm ||.||_frob -// Where ||X||_frob=sqrt[sum_ij(x_ij^2)] (basically just turns an n,n matrix into vector in n^2 space -// then does Euclidean norm) -void ParameterHandlerBase::MakeClosestPosDef(TMatrixDSym *cov) { -// ******************************************** - // Want to get cov' = (cov_sym+cov_polar)/2 - // cov_sym=(cov+cov^T)/2 - // cov_polar-> SVD cov to cov=USV^T then cov_polar=VSV^T - - //Get frob norm of cov - // Double_t cov_norm=cov->E2Norm(); - - TMatrixDSym* cov_trans = cov; - cov_trans->T(); - TMatrixDSym cov_sym = 0.5*(*cov+*cov_trans); //If cov is symmetric does nothing, otherwise just ensures symmetry - - //Do SVD to get polar form - TDecompSVD cov_sym_svd=TDecompSVD(cov_sym); - if(!cov_sym_svd.Decompose()){ - MACH3LOG_WARN("Cannot do SVD on input matrix, trying MakePosDef() first!"); - MakePosDef(&cov_sym); - } - - TMatrixD cov_sym_v = cov_sym_svd.GetV(); - TMatrixD cov_sym_vt = cov_sym_v; - cov_sym_vt.T(); - //SVD returns as vector (grrr) so need to get into matrix form for multiplying! - TVectorD cov_sym_sigvect = cov_sym_svd.GetSig(); - - const Int_t nCols = cov_sym_v.GetNcols(); //square so only need rows hence lack of cols - TMatrixDSym cov_sym_sig(nCols); - TMatrixDDiag cov_sym_sig_diag(cov_sym_sig); - cov_sym_sig_diag=cov_sym_sigvect; - - //Can finally get H=VSV - TMatrixDSym cov_sym_polar = cov_sym_sig.SimilarityT(cov_sym_vt);//V*S*V^T (this took forver to find!) - - //Now we can construct closest approximater Ahat=0.5*(B+H) - TMatrixDSym cov_closest_approx = 0.5*(cov_sym+cov_sym_polar);//Not fully sure why this is even needed since symmetric B -> U=V - //Get norm of transformed - // Double_t approx_norm=cov_closest_approx.E2Norm(); - //MACH3LOG_INFO("Initial Norm: {:.6f} | Norm after transformation: {:.6f} | Ratio: {:.6f}", cov_norm, approx_norm, cov_norm / approx_norm); - - *cov = cov_closest_approx; - //Now can just add a makeposdef! - MakePosDef(cov); -} - -// ******************************************** -// KS: Convert covariance matrix to correlation matrix and return TH2D which can be used for fancy plotting -TH2D* ParameterHandlerBase::GetCorrelationMatrix() { -// ******************************************** - TH2D* hMatrix = new TH2D(GetName().c_str(), GetName().c_str(), _fNumPar, 0.0, _fNumPar, _fNumPar, 0.0, _fNumPar); - hMatrix->SetDirectory(nullptr); - for(int i = 0; i < _fNumPar; i++) - { - hMatrix->SetBinContent(i+1, i+1, 1.); - hMatrix->GetXaxis()->SetBinLabel(i+1, GetParFancyName(i).c_str()); - hMatrix->GetYaxis()->SetBinLabel(i+1, GetParFancyName(i).c_str()); - } - - #ifdef MULTITHREAD - #pragma omp parallel for - #endif - for(int i = 0; i < _fNumPar; i++) - { - for(int j = 0; j <= i; j++) - { - const double Corr = (*covMatrix)(i,j) / ( GetDiagonalError(i) * GetDiagonalError(j)); - hMatrix->SetBinContent(i+1, j+1, Corr); - hMatrix->SetBinContent(j+1, i+1, Corr); - } - } - return hMatrix; -} - -// ******************************************** -// KS: After step scale, prefit etc. value were modified save this modified config. -void ParameterHandlerBase::SaveUpdatedMatrixConfig() { -// ******************************************** - if (!_fYAMLDoc) - { - MACH3LOG_CRITICAL("Yaml node hasn't been initialised for matrix {}, something is not right", matrixName); +// KS: After step scale, prefit etc. value were modified save this modified +// config. +void ParameterHandlerBase::SaveUpdatedMatrixConfig( + std::string const &filename) { + // ******************************************** + if (!config.YAMLDoc) { + MACH3LOG_CRITICAL("Yaml node hasn't been initialised for matrix {}, " + "something is not right", + matrixName); MACH3LOG_CRITICAL("I am not throwing error but should be investigated"); return; } - YAML::Node copyNode = _fYAMLDoc; + YAML::Node copyNode = config.YAMLDoc; int i = 0; - for (YAML::Node param : copyNode["Systematics"]) - { - //KS: Feel free to update it, if you need updated prefit value etc - param["Systematic"]["StepScale"]["MCMC"] = MaCh3Utils::FormatDouble(_fIndivStepScale[i], 4); + for (YAML::Node param : copyNode["Systematics"]) { + // KS: Feel free to update it, if you need updated prefit value etc + param["Systematic"]["StepScale"]["MCMC"] = + MaCh3Utils::FormatDouble(params.stepscale[i], 4); i++; } // Save the modified node to a file - std::ofstream fout("Modified_Matrix.yaml"); + std::ofstream fout(filename); fout << copyNode; fout.close(); } // ******************************************** -bool ParameterHandlerBase::AppliesToSample(const int SystIndex, const std::string& SampleName) const { -// ******************************************** +bool ParameterHandlerBase::AppliesToSample( + const int SystIndex, const std::string &SampleName) const { + // ******************************************** // Empty means apply to all - if (_fSampleNames[SystIndex].size() == 0) return true; + if (!params.samples[SystIndex].size()) { + return true; + } // Make a copy and to lower case to not be case sensitive std::string SampleNameCopy = SampleName; - std::transform(SampleNameCopy.begin(), SampleNameCopy.end(), SampleNameCopy.begin(), ::tolower); + std::transform(SampleNameCopy.begin(), SampleNameCopy.end(), + SampleNameCopy.begin(), ::tolower); // Check for unsupported wildcards in SampleNameCopy if (SampleNameCopy.find('*') != std::string::npos) { - MACH3LOG_ERROR("Wildcards ('*') are not supported in sample name: '{}'", SampleName); - throw MaCh3Exception(__FILE__ , __LINE__ ); + MACH3LOG_ERROR("Wildcards ('*') are not supported in sample name: '{}'", + SampleName); + throw MaCh3Exception(__FILE__, __LINE__); } bool Applies = false; @@ -1328,14 +1107,15 @@ bool ParameterHandlerBase::AppliesToSample(const int SystIndex, const std::strin std::transform(pattern.begin(), pattern.end(), pattern.begin(), ::tolower); // Replace '*' in the pattern with '.*' for regex matching - std::string regexPattern = "^" + std::regex_replace(pattern, std::regex("\\*"), ".*") + "$"; + std::string regexPattern = + "^" + std::regex_replace(pattern, std::regex("\\*"), ".*") + "$"; try { std::regex regex(regexPattern); if (std::regex_match(SampleNameCopy, regex)) { Applies = true; break; } - } catch (const std::regex_error& e) { + } catch (const std::regex_error &e) { // Handle regex error (for invalid patterns) MACH3LOG_ERROR("Regex error: {}", e.what()); } @@ -1345,15 +1125,9 @@ bool ParameterHandlerBase::AppliesToSample(const int SystIndex, const std::strin // ******************************************** // Set proposed parameter values vector to be base on tune values -void ParameterHandlerBase::SetTune(const std::string& TuneName) { -// ******************************************** - if(Tunes == nullptr) { - MACH3LOG_ERROR("Tunes haven't been initialised, which are being loaded from YAML, have you used some deprecated constructor"); - throw MaCh3Exception(__FILE__, __LINE__); - } - auto Values = Tunes->GetTune(TuneName); - - SetParameters(Values); +void ParameterHandlerBase::SetTune(const std::string &TuneName) { + // ******************************************** + SetParameters(Tunes->GetTune(TuneName)); } // ************************************* @@ -1361,25 +1135,27 @@ void ParameterHandlerBase::SetTune(const std::string& TuneName) { /// /// @param PosteriorFile Pointer to the ROOT TTree from MaCh3 fit. /// @param Systematic Pointer to the systematic parameter handler. -/// @param[out] BranchValues Vector to store the values of the branches (resized inside). -/// @param[out] BranchNames Vector to store the names of the branches (resized inside). +/// @param[out] BranchValues Vector to store the values of the branches (resized +/// inside). +/// @param[out] BranchNames Vector to store the names of the branches (resized +/// inside). /// /// @throws MaCh3Exception if any parameter branch is uninitialized. -void ParameterHandlerBase::MatchMaCh3OutputBranches(TTree *PosteriorFile, - std::vector& BranchValues, - std::vector& BranchNames) { -// ************************************* +void ParameterHandlerBase::MatchMaCh3OutputBranches( + TTree *PosteriorFile, std::vector &BranchValues, + std::vector &BranchNames) { + // ************************************* BranchValues.resize(GetNumParams()); BranchNames.resize(GetNumParams()); for (int i = 0; i < GetNumParams(); ++i) { BranchNames[i] = GetParName(i); if (!PosteriorFile->GetBranch(BranchNames[i].c_str())) { - MACH3LOG_ERROR("Branch '{}' does not exist in the TTree!", BranchNames[i]); + MACH3LOG_ERROR("Branch '{}' does not exist in the TTree!", + BranchNames[i]); throw MaCh3Exception(__FILE__, __LINE__); } PosteriorFile->SetBranchStatus(BranchNames[i].c_str(), true); PosteriorFile->SetBranchAddress(BranchNames[i].c_str(), &BranchValues[i]); } } - diff --git a/Parameters/ParameterHandlerBase.h b/Parameters/ParameterHandlerBase.h index 263184503..a36cb0d8c 100644 --- a/Parameters/ParameterHandlerBase.h +++ b/Parameters/ParameterHandlerBase.h @@ -2,52 +2,87 @@ // MaCh3 includes #include "Manager/Manager.h" -#include "Parameters/ParameterHandlerUtils.h" #include "Parameters/AdaptiveMCMCHandler.h" #include "Parameters/PCAHandler.h" +#include "Parameters/ParameterHandlerUtils.h" #include "Parameters/ParameterTunes.h" -/// @brief Base class responsible for handling of systematic error parameters. Capable of using PCA or using adaptive throw matrix -/// @see For more details, visit the [Wiki](https://github.com/mach3-software/MaCh3/wiki/02.-Implementation-of-Systematic). +#include "Eigen/Dense" + +/// @brief Base class responsible for handling of systematic error parameters. +/// Capable of using PCA or using adaptive throw matrix +/// @see For more details, visit the +/// [Wiki](https://github.com/mach3-software/MaCh3/wiki/02.-Implementation-of-Systematic). /// @author Dan Barrow /// @author Ed Atkin /// @author Kamil Skwarczynski class ParameterHandlerBase { - public: +public: + struct ParamInfo { + std::string name, fancy_name; + double prefit, error, stepscale; + std::array bounds; + bool flatprior, fixed; + std::vector affected_samples; + }; + + void AddParameters(std::vector const ¶ms); + void AddParameter(ParamInfo const ¶m) { + AddParameters({ + params, + }); + } + /// @brief ETA - constructor for a YAML file - /// @param YAMLFile A vector of strings representing the YAML files used for initialisation of matrix + /// @param YAMLFile A vector of strings representing the YAML files used for + /// initialisation of matrix /// @param name Matrix name /// @param threshold PCA threshold from 0 to 1. Default is -1 and means no PCA /// @param FirstPCAdpar First PCA parameter that will be decomposed. /// @param LastPCAdpar First PCA parameter that will be decomposed. - ParameterHandlerBase(const std::vector& YAMLFile, std::string name, double threshold = -1, int FirstPCAdpar = -999, int LastPCAdpar = -999); + static ParameterHandlerBase + MakeFromYAML(const std::vector &YAMLFiles, std::string name, + double threshold = -1, int FirstPCAdpar = -999, + int LastPCAdpar = -999) { + return ParameterHandlerBase(YAMLFiles, name, threshold, FirstPCAdpar, + LastPCAdpar); + } /// @brief "Usual" constructors from root file /// @param name Matrix name /// @param file Path to matrix root file - ParameterHandlerBase(std::string name, std::string file, double threshold = -1, int FirstPCAdpar = -999, int LastPCAdpar = -999); + static ParameterHandlerBase + MakeFromTMatrix(std::string name, std::string file, double threshold = -1, + int FirstPCAdpar = -999, int LastPCAdpar = -999) { + return ParameterHandlerBase(name, file, threshold, FirstPCAdpar, + LastPCAdpar); + } /// @brief Destructor - virtual ~ParameterHandlerBase(); - + virtual ~ParameterHandlerBase() {}; + /// @defgroup ParameterHandlerSetters Parameter Handler Setters /// Group of functions to set various parameters, names, and values. /// @defgroup ParameterHandlerGetters Parameter Handler Getters /// Group of functions to get various parameters, names, and values. - // ETA - maybe need to add checks to index on the setters? i.e. if( i > _fPropVal.size()){throw;} + // ETA - maybe need to add checks to index on the setters? i.e. if( i > + // _fPropVal.size()){throw;} /// @brief Set covariance matrix - /// @param cov Covariance matrix which we set and will be used later for evaluation of penalty term + /// @param cov Covariance matrix which we set and will be used later for + /// evaluation of penalty term /// @ingroup ParameterHandlerSetters void SetCovMatrix(TMatrixDSym *cov); /// @brief Set matrix name /// @ingroup ParameterHandlerSetters - void SetName(const std::string& name) { matrixName = name; } + void SetName(const std::string &name) { settings.name = name; } /// @brief change parameter name /// @param i Parameter index /// @param name new name which will be set /// @ingroup ParameterHandlerSetters - void SetParName(const int i, const std::string& name) { _fNames.at(i) = name; } + void SetParName(const int i, const std::string &name) { + params.name.at(i) = name; + } /// @brief Set value of single param to a given value /// @ingroup ParameterHandlerSetters void SetSingleParameter(const int parNo, const double parVal); @@ -65,67 +100,81 @@ class ParameterHandlerBase { /// @param i Parameter index /// @param val new value which will be set /// @ingroup ParameterHandlerSetters - void SetParProp(const int i, const double val) { - _fPropVal[i] = val; - if (pca) PCAObj->TransferToPCA(); - } - /// @brief Set parameter values using vector, it has to have same size as covariance class + void SetParProp(const int i, const double val) { params.proposed[i] = val; } + /// @brief Set parameter values using vector, it has to have same size as + /// covariance class /// @param pars Vector holding new values for every parameter /// @ingroup ParameterHandlerSetters - void SetParameters(const std::vector& pars = {}); + void SetParameters(const std::vector &pars = {}); /// @brief Set if parameter should have flat prior or not /// @param i Parameter index /// @param eL bool telling if it will be flat or not /// @ingroup ParameterHandlerSetters void SetFlatPrior(const int i, const bool eL); - + /// @brief Set random value useful for debugging/CI /// @param i Parameter index /// @param rand New value for random number /// @ingroup ParameterHandlerSetters - void SetRandomThrow(const int i, const double rand) { randParams[i] = rand;} + void SetRandomThrow(const int i, const double rand) { + throws.random_vector[i] = rand; + } /// @brief Get random value useful for debugging/CI /// @param i Parameter index /// @ingroup ParameterHandlerGetters - double GetRandomThrow(const int i) const { return randParams[i];} + double GetRandomThrow(const int i) const { return throws.random_vector[i]; } /// @brief set branches for output file /// @param tree Tree to which we will save branches - /// @param SaveProposal Normally we only save parameter after is accepted, for debugging purpose it is helpful to see also proposed values. That's what this variable controls + /// @param SaveProposal Normally we only save parameter after is accepted, for + /// debugging purpose it is helpful to see also proposed values. That's what + /// this variable controls /// @ingroup ParameterHandlerSetters void SetBranches(TTree &tree, const bool SaveProposal = false); /// @brief Set global step scale for covariance object /// @param scale Value of global step scale - /// @param verbose Print that we've changed scale + use warnings [default: true] + /// @param verbose Print that we've changed scale + use warnings [default: + /// true] /// @cite luengo2020survey /// @ingroup ParameterHandlerSetters - void SetStepScale(const double scale, const bool verbose=true); - /// @brief DB Function to set fIndivStepScale from a vector (Can be used from execs and inside covariance constructors) + void SetStepScale(const double scale, const bool verbose = true); + /// @brief DB Function to set fIndivStepScale from a vector (Can be used from + /// execs and inside covariance constructors) /// @param ParameterIndex Parameter Index /// @param StepScale Value of individual step scale /// @ingroup ParameterHandlerSetters - void SetIndivStepScale(const int ParameterIndex, const double StepScale){ _fIndivStepScale.at(ParameterIndex) = StepScale; } - /// @brief DB Function to set fIndivStepScale from a vector (Can be used from execs and inside covariance constructors) + void SetIndivStepScale(const int ParameterIndex, const double StepScale) { + steps.scale[ParameterIndex] = StepScale; + } + /// @brief DB Function to set fIndivStepScale from a vector (Can be used from + /// execs and inside covariance constructors) /// @param stepscale Vector of individual step scale, should have same /// @ingroup ParameterHandlerSetters - void SetIndivStepScale(const std::vector& stepscale); + void SetIndivStepScale(const std::vector &stepscale); /// @brief KS: In case someone really want to change this /// @ingroup ParameterHandlerSetters - inline void SetPrintLength(const unsigned int PriLen) { PrintLength = PriLen; } + inline void SetPrintLength(const unsigned int PriLen) { + settings.PrintLength = PriLen; + } - /// @brief KS: After step scale, prefit etc. value were modified save this modified config. - void SaveUpdatedMatrixConfig(); + /// @brief KS: After step scale, prefit etc. value were modified save this + /// modified config. + void + SaveUpdatedMatrixConfig(std::string const &filename = "Modified_Matrix.yaml"); - /// @brief Throw the proposed parameter by mag sigma. Should really just have the user specify this throw by having argument double + /// @brief Throw the proposed parameter by mag sigma. Should really just have + /// the user specify this throw by having argument double void ThrowParProp(const double mag = 1.); - /// @brief Helper function to throw the current parameter by mag sigma. Can study bias in MCMC with this; put different starting parameters + /// @brief Helper function to throw the current parameter by mag sigma. Can + /// study bias in MCMC with this; put different starting parameters void ThrowParCurr(const double mag = 1.); - /// @brief Throw the parameters according to the covariance matrix. This shouldn't be used in MCMC code ase it can break Detailed Balance; + /// @brief Throw the parameters according to the covariance matrix. This + /// shouldn't be used in MCMC code ase it can break Detailed Balance; void ThrowParameters(); /// @brief Randomly throw the parameters in their 1 sigma range void RandomConfiguration(); - + /// @brief Check if parameters were proposed outside physical boundary int CheckBounds() const _noexcept_; /// @brief Calc penalty term based on inverted covariance matrix @@ -133,28 +182,35 @@ class ParameterHandlerBase { /// @details /// The log-likelihood is computed as: /// \f[ - /// \log \mathcal{L} = \frac{1}{2} \sum_{i}^{\textrm{pars}} \sum_{j}^{\textrm{pars}} \Delta \vec{p}_i \left( V^{-1} \right)_{i,j} \Delta \vec{p}_j + /// \log \mathcal{L} = \frac{1}{2} \sum_{i}^{\textrm{pars}} + /// \sum_{j}^{\textrm{pars}} \Delta \vec{p}_i \left( V^{-1} \right)_{i,j} + /// \Delta \vec{p}_j /// \f] /// where: - /// - \f$\Delta \vec{p}_i = \theta_i - \theta_{i,0}\f$ is the difference between the current and pre-fit parameter values, + /// - \f$\Delta \vec{p}_i = \theta_i - \theta_{i,0}\f$ is the difference + /// between the current and pre-fit parameter values, /// - \f$V^{-1}\f$ is the inverted covariance matrix. /// /// @note - /// - If `_fFlatPrior[i]` is `true`, the parameter is excluded from the calculation. + /// - If `_fFlatPrior[i]` is `true`, the parameter is excluded from the + /// calculation. double CalcLikelihood() const _noexcept_; - /// @brief Return CalcLikelihood if some params were thrown out of boundary return _LARGE_LOGL_ + /// @brief Return CalcLikelihood if some params were thrown out of boundary + /// return _LARGE_LOGL_ /// @ingroup ParameterHandlerGetters virtual double GetLikelihood(); /// @brief Return covariance matrix /// @ingroup ParameterHandlerGetters - TMatrixDSym *GetCovMatrix() const { return covMatrix; } + TMatrixDSym *GetCovMatrix() const { return &root_copies.covariance; } /// @brief Return inverted covariance matrix /// @ingroup ParameterHandlerGetters - TMatrixDSym *GetInvCovMatrix() const { return invCovMatrix; } + TMatrixDSym *GetInvCovMatrix() const { return &root_copies.inv_covariance; } /// @brief Return inverted covariance matrix /// @ingroup ParameterHandlerGetters - double GetInvCovMatrix(const int i, const int j) const { return InvertCovMatrix[i][j]; } + double GetInvCovMatrix(const int i, const int j) const { + return params.inv_covariance(i, j); + } /// @brief Return correlated throws /// @param i Parameter index @@ -164,137 +220,140 @@ class ParameterHandlerBase { /// @brief Get if param has flat prior or not /// @param i Parameter index /// @ingroup ParameterHandlerGetters - inline bool GetFlatPrior(const int i) const { return _fFlatPrior[i]; } + inline bool GetFlatPrior(const int i) const { return params.flatprior[i]; } /// @brief Get name of covariance /// @ingroup ParameterHandlerGetters - std::string GetName() const { return matrixName; } + std::string GetName() const { return params.matrix_name; } /// @brief Get name of parameter /// @param i Parameter index /// @ingroup ParameterHandlerGetters - std::string GetParName(const int i) const {return _fNames[i];} + std::string GetParName(const int i) const { return params.name[i]; } /// @brief Get index based on name /// @ingroup ParameterHandlerGetters - int GetParIndex(const std::string& name) const; + int GetParIndex(const std::string &name) const; /// @brief Get fancy name of the Parameter /// @param i Parameter index /// @ingroup ParameterHandlerGetters - std::string GetParFancyName(const int i) const {return _fFancyNames[i];} + std::string GetParFancyName(const int i) const { return params.fancyname[i]; } /// @brief Get name of input file /// @ingroup ParameterHandlerGetters - std::string GetInputFile() const { return inputFile; } + std::string GetInputFile() const { return config.inputFile; } /// @brief Get diagonal error for ith parameter /// @param i Parameter index /// @ingroup ParameterHandlerGetters - inline double GetDiagonalError(const int i) const { return std::sqrt((*covMatrix)(i,i)); } + inline double GetDiagonalError(const int i) const { + return std::sqrt(params.covariance(i, i)); + } /// @brief Get the error for the ith parameter /// @param i Parameter index /// @ingroup ParameterHandlerGetters - inline double GetError(const int i) const {return _fError[i];} + inline double GetError(const int i) const { return params.error[i]; } /// @brief Adaptive Step Tuning Stuff void ResetIndivStepScale(); /// @brief Initialise adaptive MCMC /// @param adapt_manager Node having from which we load all adaptation options - void InitialiseAdaption(const YAML::Node& adapt_manager); + void InitialiseAdaption(const YAML::Node &adapt_manager); /// @brief Save adaptive throw matrix to file - void SaveAdaptiveToFile(const std::string& outFileName, const std::string& systematicName) { - AdaptiveHandler->SaveAdaptiveToFile(outFileName, systematicName); } + void SaveAdaptiveToFile(const std::string &outFileName, + const std::string &systematicName) { + AdaptiveHandler.SaveAdaptiveToFile(outFileName, systematicName); + } /// @brief Do we adapt or not /// @ingroup ParameterHandlerGetters - bool GetDoAdaption() const {return use_adaptive;} + bool GetDoAdaption() const { return settings.use_adaptive; } /// @brief Use new throw matrix, used in adaptive MCMC /// @ingroup ParameterHandlerSetters void SetThrowMatrix(TMatrixDSym *cov); /// @brief Replaces old throw matrix with new one void UpdateThrowMatrix(TMatrixDSym *cov); - /// @brief Set number of MCMC step, when running adaptive MCMC it is updated with given frequency. We need number of steps to determine frequency. - /// @ingroup ParameterHandlerSetters + /// @brief Set number of MCMC step, when running adaptive MCMC it is updated + /// with given frequency. We need number of steps to determine frequency. + /// @ingroup ParameterHandlerSetters inline void SetNumberOfSteps(const int nsteps) { - AdaptiveHandler->SetTotalSteps(nsteps); - if(AdaptiveHandler->AdaptionUpdate()) ResetIndivStepScale(); + AdaptiveHandler.SetTotalSteps(nsteps); + if (AdaptiveHandler.AdaptionUpdate()) { + ResetIndivStepScale(); + } } /// @brief Get matrix used for step proposal /// @ingroup ParameterHandlerGetters - inline TMatrixDSym *GetThrowMatrix() const {return throwMatrix;} + inline TMatrixDSym *GetThrowMatrix() const { return &root_copies.proposal; } /// @brief Get matrix used for step proposal /// @ingroup ParameterHandlerGetters - double GetThrowMatrix(const int i, const int j) const { return throwMatrixCholDecomp[i][j];} + double GetThrowMatrix(const int i, const int j) const { + return params.l_proposal(i, j); + } - /// @brief KS: Convert covariance matrix to correlation matrix and return TH2D which can be used for fancy plotting - /// @details This function converts the covariance matrix to a correlation matrix and - /// returns a TH2D object, which can be used for advanced plotting purposes. + /// @brief KS: Convert covariance matrix to correlation matrix and return TH2D + /// which can be used for fancy plotting + /// @details This function converts the covariance matrix to a correlation + /// matrix and + /// returns a TH2D object, which can be used for advanced plotting + /// purposes. /// @return A pointer to a TH2D object representing the correlation matrix /// @ingroup ParameterHandlerGetters - TH2D* GetCorrelationMatrix(); - - /// @brief DB Pointer return to param position - /// - /// @param iParam The index of the parameter in the vector. - /// @return A pointer to the parameter value at the specified index. - /// - /// @warning ETA - This might be a bit squiffy? If the vector gots moved from say a - /// push_back then the pointer is no longer valid... maybe need a better - /// way to deal with this? It was fine before when the return was to an - /// element of a new array. There must be a clever C++ way to be careful - inline const double* RetPointer(const int iParam) {return &(_fPropVal.data()[iParam]);} - - /// @brief Get a reference to the proposed parameter values - /// Can be useful if you want to track these without having to copy values using getProposed() - inline const std::vector &GetParPropVec() {return _fPropVal;} + TH2D *GetCorrelationMatrix(); /// @brief Get total number of parameters /// @ingroup ParameterHandlerGetters - inline int GetNumParams() const {return _fNumPar;} + inline int GetNumParams() const { return params.prefit.size(); } /// @brief Get the pre-fit values of the parameters. /// @ingroup ParameterHandlerGetters - std::vector GetPreFitValues() const {return _fPreFitValue;} + std::vector GetPreFitValues() const { + return M3::EigenToStdVector(params.prefit); + } /// @brief Get vector of all proposed parameter values /// @ingroup ParameterHandlerGetters - std::vector GetProposed() const; + std::vector GetProposed() const { + return M3::EigenToStdVector(params.proposed); + } /// @brief Get proposed parameter value /// @param i Parameter index /// @ingroup ParameterHandlerGetters - inline double GetParProp(const int i) const { return _fPropVal[i]; } + inline double GetParProp(const int i) const { return params.proposed[i]; } /// @brief Get current parameter value /// @param i Parameter index /// @ingroup ParameterHandlerGetters - inline double GetParCurr(const int i) const { return _fCurrVal[i]; } - /// @brief Get vector of current parameter values - /// @ingroup ParameterHandlerGetters - inline const std::vector &GetParCurrVec() const { return _fCurrVal; } + inline double GetParCurr(const int i) const { return params.current[i]; } /// @brief Get prior parameter value /// @param i Parameter index /// @ingroup ParameterHandlerGetters - inline double GetParInit(const int i) const { return _fPreFitValue[i]; } + inline double GetParInit(const int i) const { return params.prefit[i]; } /// @brief Get upper parameter bound in which it is physically valid /// @param i Parameter index /// @ingroup ParameterHandlerGetters - inline double GetUpperBound(const int i) const { return _fUpBound[i];} + inline double GetUpperBound(const int i) const { return params.upbound[i]; } /// @brief Get lower parameter bound in which it is physically valid /// @param i Parameter index /// @ingroup ParameterHandlerGetters - inline double GetLowerBound(const int i) const { return _fLowBound[i]; } + inline double GetLowerBound(const int i) const { return params.lowbound[i]; } /// @brief Get individual step scale for selected parameter /// @param ParameterIndex Parameter index /// @ingroup ParameterHandlerGetters - inline double GetIndivStepScale(const int ParameterIndex) const {return _fIndivStepScale.at(ParameterIndex); } + inline double GetIndivStepScale(const int i) const { + return params.stepscale[i]; + } /// @brief Get global step scale for covariance object /// @ingroup ParameterHandlerGetters - inline double GetGlobalStepScale() const {return _fGlobalStepScale; } + inline double GetGlobalStepScale() const { return settings.GlobalStepScale; } - /// @brief Get number of params which will be different depending if using Eigen decomposition or not + /// @brief Get number of params which will be different depending if using + /// Eigen decomposition or not /// @ingroup ParameterHandlerGetters inline int GetNParameters() const { - if (pca) return PCAObj->GetNumberPCAedParameters(); - else return _fNumPar; + if (pca) + return PCAObj->GetNumberPCAedParameters(); + else + return GetNumParams(); } /// @brief Print prior value for every parameter @@ -303,13 +362,15 @@ class ParameterHandlerBase { void PrintNominalCurrProp() const; /// @warning only for backward compatibility /// @todo remove it - void PrintParameters() const {PrintNominalCurrProp();}; + void PrintParameters() const { PrintNominalCurrProp(); }; /// @brief Print step scale for each parameter void PrintIndivStepScale() const; /// @brief Generate a new proposed state virtual void ProposeStep(); - /// @brief "Randomize" the parameters in the covariance class for the proposed step. Used the proposal kernel and the current parameter value to set proposed step + /// @brief "Randomize" the parameters in the covariance class for the proposed + /// step. Used the proposal kernel and the current parameter value to set + /// proposed step void Randomize() _noexcept_; /// @brief Use Cholesky throw matrix for better step proposal void CorrelateSteps() _noexcept_; @@ -327,7 +388,7 @@ class ParameterHandlerBase { void SetFixParameter(const int i); /// @brief Set parameter to be fixed at prior value /// @param name Name of the parameter to be fixed - void SetFixParameter(const std::string& name); + void SetFixParameter(const std::string &name); /// @brief Set all parameters to be treated as free void SetFreeAllParameters(); @@ -336,7 +397,7 @@ class ParameterHandlerBase { void SetFreeParameter(const int i); /// @brief Set parameter to be treated as free /// @param name Name of the parameter to be treated as free - void SetFreeParameter(const std::string& name); + void SetFreeParameter(const std::string &name); /// @brief Toggle fixing parameters at prior values void ToggleFixAllParameters(); @@ -345,187 +406,193 @@ class ParameterHandlerBase { void ToggleFixParameter(const int i); /// @brief Toggle fixing parameter at prior values /// @param name Name of parameter you want to fix - void ToggleFixParameter(const std::string& name); + void ToggleFixParameter(const std::string &name); /// @brief Is parameter fixed or not /// @param i Parameter index - bool IsParameterFixed(const int i) const { - if (_fError[i] < 0) { return true; } - else { return false; } - } + bool IsParameterFixed(const int i) const { return params.fixed[i]; } + /// @brief Is parameter fixed or not /// @param name Name of parameter you want to check if is fixed - bool IsParameterFixed(const std::string& name) const; + bool IsParameterFixed(const std::string &name) const; - /// @brief CW: Calculate eigen values, prepare transition matrices and remove param based on defined threshold - /// @param eigen_threshold PCA threshold from 0 to 1. Default is -1 and means no PCA + /// @brief CW: Calculate eigen values, prepare transition matrices and remove + /// param based on defined threshold + /// @param eigen_threshold PCA threshold from 0 to 1. Default is -1 and means + /// no PCA /// @param FirstPCAdpar First PCA parameter that will be decomposed. /// @param LastPCAdpar First PCA parameter that will be decomposed. - /// @see For more details, visit the [Wiki](https://github.com/mach3-software/MaCh3/wiki/03.-Eigen-Decomposition-%E2%80%90-PCA). - void ConstructPCA(const double eigen_threshold, int FirstPCAdpar, int LastPCAdpar); + /// @see For more details, visit the + /// [Wiki](https://github.com/mach3-software/MaCh3/wiki/03.-Eigen-Decomposition-%E2%80%90-PCA). + void ConstructPCA(const double eigen_threshold, int FirstPCAdpar, + int LastPCAdpar); /// @brief is PCA, can use to query e.g. LLH scans inline bool IsPCA() const { return pca; } /// @brief Getter to return a copy of the YAML node /// @ingroup ParameterHandlerGetters - YAML::Node GetConfig() const { return _fYAMLDoc; } + YAML::Node GetConfig() const { return config.YAMLDoc; } /// @brief Get pointer for AdaptiveHandler /// @ingroup ParameterHandlerGetters - inline adaptive_mcmc::AdaptiveMCMCHandler* GetAdaptiveHandler() const { + inline adaptive_mcmc::AdaptiveMCMCHandler &GetAdaptiveHandler() const { if (!use_adaptive) { MACH3LOG_ERROR("Am not running in Adaptive mode"); - throw MaCh3Exception(__FILE__ , __LINE__ ); + throw MaCh3Exception(__FILE__, __LINE__); } - return AdaptiveHandler.get(); + return AdaptiveHandler; } - /// @brief KS: Set proposed parameter values vector to be base on tune values, for example set proposed values to be of generated or maybe PostND + /// @brief KS: Set proposed parameter values vector to be base on tune values, + /// for example set proposed values to be of generated or maybe PostND /// @ingroup ParameterHandlerSetters - void SetTune(const std::string& TuneName); - + void SetTune(const std::string &TuneName); + /// @brief Get pointer for PCAHandler - inline PCAHandler* GetPCAHandler() const { + inline PCAHandler &GetPCAHandler() const { if (!pca) { MACH3LOG_ERROR("Am not running in PCA mode"); - throw MaCh3Exception(__FILE__ , __LINE__ ); + throw MaCh3Exception(__FILE__, __LINE__); } - return PCAObj.get(); + return PCAObj; } /// @brief Matches branches in a TTree to parameters in a systematic handler. /// /// @param PosteriorFile Pointer to the ROOT TTree from MaCh3 fit. - /// @param[out] BranchValues Vector to store the values of the branches (resized inside). - /// @param[out] BranchNames Vector to store the names of the branches (resized inside). + /// @param[out] BranchValues Vector to store the values of the branches + /// (resized inside). + /// @param[out] BranchNames Vector to store the names of the branches (resized + /// inside). /// /// @throws MaCh3Exception if any parameter branch is uninitialized. void MatchMaCh3OutputBranches(TTree *PosteriorFile, - std::vector& BranchValues, - std::vector& BranchNames); + std::vector &BranchValues, + std::vector &BranchNames); protected: - /// @brief Initialisation of the class using matrix from root file - void Init(const std::string& name, const std::string& file); - /// @brief Initialisation of the class using config - /// @param YAMLFile A vector of strings representing the YAML files used for initialisation of matrix - void Init(const std::vector& YAMLFile); - /// @brief Initialise vectors with parameters information - /// @param size integer telling size to which we will resize all vectors/allocate memory - void ReserveMemory(const int size); - - /// @brief Make matrix positive definite by adding small values to diagonal, necessary for inverting matrix - /// @param cov Matrix which we evaluate Positive Definitiveness - void MakePosDef(TMatrixDSym *cov = nullptr); - - /// @brief HW: Finds closest possible positive definite matrix in Frobenius Norm ||.||_frob Where ||X||_frob=sqrt[sum_ij(x_ij^2)] (basically just turns an n,n matrix into vector in n^2 space then does Euclidean norm) - void MakeClosestPosDef(TMatrixDSym *cov); + ParameterHandlerBase(); + + /// @brief ETA - constructor for a YAML file + /// @param YAMLFile A vector of strings representing the YAML files used for + /// initialisation of matrix + /// @param name Matrix name + /// @param threshold PCA threshold from 0 to 1. Default is -1 and means no PCA + /// @param FirstPCAdpar First PCA parameter that will be decomposed. + /// @param LastPCAdpar First PCA parameter that will be decomposed. + ParameterHandlerBase(const std::vector &YAMLFiles, + std::string name, double threshold = -1, + int FirstPCAdpar = -999, int LastPCAdpar = -999); + /// @brief "Usual" constructors from root file + /// @param name Matrix name + /// @param file Path to matrix root file + ParameterHandlerBase(std::string name, std::string file, + double threshold = -1, int FirstPCAdpar = -999, + int LastPCAdpar = -999); /// @brief sets throw matrix from a file /// @param matrix_file_name name of file matrix lives in /// @param matrix_name name of matrix in file /// @param means_name name of means vec in file - void SetThrowMatrixFromFile(const std::string& matrix_file_name, const std::string& matrix_name, const std::string& means_name); + void SetThrowMatrixFromFile(const std::string &matrix_file_name, + const std::string &matrix_name, + const std::string &means_name); /// @brief Check if parameter is affecting given sample name /// @param SystIndex number of parameter /// @param SampleName The Sample name used to filter parameters. - bool AppliesToSample(const int SystIndex, const std::string& SampleName) const; + bool AppliesToSample(const int SystIndex, + const std::string &SampleName) const; - /// @brief KS: Flip parameter around given value, for example mass ordering around 0 + /// @brief KS: Flip parameter around given value, for example mass ordering + /// around 0 /// @param index parameter index you want to flip /// @param FlipPoint Value around which flipping is done void FlipParameterValue(const int index, const double FlipPoint); - /// @brief HW :: This method is a tad hacky but modular arithmetic gives me a headache. + /// @brief HW :: This method is a tad hacky but modular arithmetic gives me a + /// headache. /// @author Henry Wallace - void CircularParBounds(const int i, const double LowBound, const double UpBound); + void CircularParBounds(const int i, const double LowBound, + const double UpBound); /// @brief Enable special proposal - void EnableSpecialProposal(const YAML::Node& param, const int Index); + void EnableSpecialProposal(const YAML::Node ¶m, const int Index); /// @brief Perform Special Step Proposal /// @warning KS: Following Asher comment we do "Step->Circular Bounds->Flip" void SpecialStepProposal(); - /// Check if any of special step proposal were enabled - bool doSpecialStepProposal; - - /// The input root file we read in - const std::string inputFile; - - /// Name of cov matrix - std::string matrixName; - /// The covariance matrix - TMatrixDSym *covMatrix; - /// The inverse covariance matrix - TMatrixDSym *invCovMatrix; - /// KS: Same as above but much faster as TMatrixDSym cache miss - std::vector> InvertCovMatrix; - - /// KS: Set Random numbers for each thread so each thread has different seed - std::vector> random_number; - - /// Random number taken from gaussian around prior error used for corr_throw - double* randParams; - /// Result of multiplication of Cholesky matrix and randParams - double* corr_throw; - /// Global step scale applied to all params in this class - double _fGlobalStepScale; - - /// KS: This is used when printing parameters, sometimes we have super long parameters name, we want to flexibly adjust couts - int PrintLength; - - /// ETA _fNames is set automatically in the covariance class to be something like xsec_i, this is currently to make things compatible with the Diagnostic tools - std::vector _fNames; - /// Fancy name for example rather than xsec_0 it is MAQE, useful for human reading - std::vector _fFancyNames; - /// Stores config describing systematics - YAML::Node _fYAMLDoc; - /// Number of systematic parameters - int _fNumPar; - /// Parameter value dictated by the prior model. Based on it penalty term is calculated - std::vector _fPreFitValue; - /// Current value of the parameter - std::vector _fCurrVal; - /// Proposed value of the parameter - std::vector _fPropVal; - /// Prior error on the parameter - std::vector _fError; - /// Lowest physical bound, parameter will not be able to go beyond it - std::vector _fLowBound; - /// Upper physical bound, parameter will not be able to go beyond it - std::vector _fUpBound; - /// Individual step scale used by MCMC algorithm - std::vector _fIndivStepScale; - /// Whether to apply flat prior or not - std::vector _fFlatPrior; - /// Tells to which samples object param should be applied - std::vector> _fSampleNames; - - /// Matrix which we use for step proposal before Cholesky decomposition (not actually used for step proposal) - TMatrixDSym* throwMatrix; - /// Throw matrix that is being used in the fit, much faster as TMatrixDSym cache miss - double** throwMatrixCholDecomp; - - /// perform PCA or not - bool pca; - /// Are we using AMCMC? - bool use_adaptive; - - /// Struct containing information about PCA - std::unique_ptr PCAObj; + struct { + bool enabled; + std::array block_indices; + Eigen::MatrixXd rotation; + } pca; + + struct { + Eigen::VectorXd random_vector, values; + } throws; + + struct { + Eigen::VectorXd current, proposed, scale; + double global_scale; + Eigen::MatrixXd l_proposal; + } steps; + + struct { + bool enabled; + /// Indices of parameters with flip symmetry + std::vector FlipParameterIndex; + /// Central points around which parameters are flipped + std::vector FlipParameterPoint; + /// Indices of parameters with circular bounds + std::vector CircularBoundsIndex; + /// Circular bounds for each parameter (lower, upper) + std::vector> CircularBoundsValues; + } special_proposal; + + struct { + std::vector name, fancy_name; + Eigen::VectorXd prefit, error, lowbound, upbound; + Eigen::VectorXi flatprior, fixed; + std::vector> samples; + + Eigen::MatrixXd covariance; + Eigen::MatrixXd inv_covariance; + } params; + + struct { + std::ranlux48 e1; + std::normal_distribution gaus; + } rng; + + // to retain external interface where possible + struct { + TMatrixDSym covariance; + TMatrixDSym inv_covariance; + TMatrixDSym proposal; + } root_copies; + + struct { + std::string name; + + /// Are we using AMCMC? + bool use_adaptive; + + /// KS: This is used when printing parameters, sometimes we have super long + /// parameters name, we want to flexibly adjust couts + int PrintLength; + } settings; + + struct { + /// The input root file we read in + std::string inputFile; + /// Stores config describing systematics + YAML::Node YAMLDoc; + } config; + /// Struct containing information about adaption - std::unique_ptr AdaptiveHandler; + adaptive_mcmc::AdaptiveMCMCHandler AdaptiveHandler; /// Struct containing information about adaption - std::unique_ptr Tunes; - - /// Indices of parameters with flip symmetry - std::vector FlipParameterIndex; - /// Central points around which parameters are flipped - std::vector FlipParameterPoint; - /// Indices of parameters with circular bounds - std::vector CircularBoundsIndex; - /// Circular bounds for each parameter (lower, upper) - std::vector> CircularBoundsValues; + ParameterTunes Tunes; }; diff --git a/Parameters/ParameterHandlerUtils.h b/Parameters/ParameterHandlerUtils.h index 0b1442466..aa2916e1c 100644 --- a/Parameters/ParameterHandlerUtils.h +++ b/Parameters/ParameterHandlerUtils.h @@ -5,539 +5,767 @@ _MaCh3_Safe_Include_Start_ //{ // ROOT includes -#include "TMatrixT.h" -#include "TMatrixDSym.h" -#include "TVectorT.h" -#include "TVectorD.h" +#include "TDecompChol.h" +#include "TDecompSVD.h" +#include "TFile.h" #include "TH1D.h" #include "TH2D.h" -#include "TTree.h" -#include "TFile.h" -#include "TRandom3.h" +#include "TKey.h" #include "TMath.h" -#include "TDecompChol.h" -#include "TStopwatch.h" #include "TMatrix.h" -#include "TMatrixDSymEigen.h" #include "TMatrixDEigen.h" -#include "TDecompSVD.h" -#include "TKey.h" -_MaCh3_Safe_Include_End_ //} - -namespace M3 -{ -/// @brief CW: Multi-threaded matrix multiplication -inline double* MatrixMult(double *A, double *B, int n) { - //CW: First transpose to increse cache hits - double *BT = new double[n*n]; - #ifdef MULTITHREAD - #pragma omp parallel for - #endif - for (int i = 0; i < n; i++) { - for (int j = 0; j < n; j++) { - BT[j*n+i] = B[i*n+j]; +#include "TMatrixDSym.h" +#include "TMatrixDSymEigen.h" +#include "TMatrixT.h" +#include "TRandom3.h" +#include "TStopwatch.h" +#include "TTree.h" +#include "TVectorD.h" +#include "TVectorT.h" + _MaCh3_Safe_Include_End_ //} + + namespace M3 { + /// @brief CW: Multi-threaded matrix multiplication + inline double *MatrixMult(double *A, double *B, int n) { + // CW: First transpose to increse cache hits + double *BT = new double[n * n]; +#ifdef MULTITHREAD +#pragma omp parallel for +#endif + for (int i = 0; i < n; i++) { + for (int j = 0; j < n; j++) { + BT[j * n + i] = B[i * n + j]; + } } - } - // Now multiply - double *C = new double[n*n]; - #ifdef MULTITHREAD - #pragma omp parallel for - #endif - for (int i = 0; i < n; i++) { - for (int j = 0; j < n; j++) { - double sum = 0; - for (int k = 0; k < n; k++) { - sum += A[i*n+k]*BT[j*n+k]; + // Now multiply + double *C = new double[n * n]; +#ifdef MULTITHREAD +#pragma omp parallel for +#endif + for (int i = 0; i < n; i++) { + for (int j = 0; j < n; j++) { + double sum = 0; + for (int k = 0; k < n; k++) { + sum += A[i * n + k] * BT[j * n + k]; + } + C[i * n + j] = sum; } - C[i*n+j] = sum; } - } - delete BT; + delete BT; - return C; -} - -/// @brief CW: Multi-threaded matrix multiplication -inline double** MatrixMult(double **A, double **B, int n) { - // First make into monolithic array - double *A_mon = new double[n*n]; - double *B_mon = new double[n*n]; + return C; + } - #ifdef MULTITHREAD - #pragma omp parallel for - #endif - for (int i = 0; i < n; ++i) { - for (int j = 0; j < n; ++j) { - A_mon[i*n+j] = A[i][j]; - B_mon[i*n+j] = B[i][j]; + /// @brief CW: Multi-threaded matrix multiplication + inline double **MatrixMult(double **A, double **B, int n) { + // First make into monolithic array + double *A_mon = new double[n * n]; + double *B_mon = new double[n * n]; + +#ifdef MULTITHREAD +#pragma omp parallel for +#endif + for (int i = 0; i < n; ++i) { + for (int j = 0; j < n; ++j) { + A_mon[i * n + j] = A[i][j]; + B_mon[i * n + j] = B[i][j]; + } } - } - //CW: Now call the monolithic calculator - double *C_mon = MatrixMult(A_mon, B_mon, n); - delete A_mon; - delete B_mon; - - // Return the double pointer - double **C = new double*[n]; - #ifdef MULTITHREAD - #pragma omp parallel for - #endif - for (int i = 0; i < n; ++i) { - C[i] = new double[n]; - for (int j = 0; j < n; ++j) { - C[i][j] = C_mon[i*n+j]; + // CW: Now call the monolithic calculator + double *C_mon = MatrixMult(A_mon, B_mon, n); + delete A_mon; + delete B_mon; + + // Return the double pointer + double **C = new double *[n]; +#ifdef MULTITHREAD +#pragma omp parallel for +#endif + for (int i = 0; i < n; ++i) { + C[i] = new double[n]; + for (int j = 0; j < n; ++j) { + C[i][j] = C_mon[i * n + j]; + } } + delete C_mon; + + return C; } - delete C_mon; - - return C; -} - -/// @brief CW: Multi-threaded matrix multiplication -inline TMatrixD MatrixMult(TMatrixD A, TMatrixD B) -{ - double *C_mon = MatrixMult(A.GetMatrixArray(), B.GetMatrixArray(), A.GetNcols()); - TMatrixD C; - C.Use(A.GetNcols(), A.GetNrows(), C_mon); - return C; -} - -// ******************************************** -/// @brief KS: Custom function to perform multiplication of matrix and vector with multithreading -/// @param VecMulti Output Vector, VecMulti = matrix x vector -/// @param matrix This matrix is used for multiplication VecMulti = matrix x vector -/// @param vector This vector is used for multiplication VecMulti = matrix x vector -/// @param n this is size of matrix and vector, we assume matrix is symmetric -inline void MatrixVectorMulti(double* _restrict_ VecMulti, double** _restrict_ matrix, const double* _restrict_ vector, const int n) { -// ******************************************** - #ifdef MULTITHREAD - #pragma omp parallel for - #endif - for (int i = 0; i < n; ++i) - { - double result = 0.0; - #ifdef MULTITHREAD - #pragma omp simd - #endif - for (int j = 0; j < n; ++j) - { - result += matrix[i][j]*vector[j]; - } - VecMulti[i] = result; - } -} - -// ******************************************** -/// @brief KS: Custom function to perform multiplication of matrix and single element which is thread safe -/// @param matrix This matrix is used for multiplication VecMulti = matrix x vector -/// @param vector This vector is used for multiplication VecMulti = matrix x vector -/// @param Length this is size of matrix and vector, we assume matrix is symmetric -/// @param i Element of matrix that we want to multiply -inline double MatrixVectorMultiSingle(double** _restrict_ matrix, const double* _restrict_ vector, const int Length, const int i) { -// ******************************************** - double Element = 0.0; - #ifdef MULTITHREAD - #pragma omp simd - #endif - for (int j = 0; j < Length; ++j) { - Element += matrix[i][j]*vector[j]; + + /// @brief CW: Multi-threaded matrix multiplication + inline TMatrixD MatrixMult(TMatrixD A, TMatrixD B) { + double *C_mon = + MatrixMult(A.GetMatrixArray(), B.GetMatrixArray(), A.GetNcols()); + TMatrixD C; + C.Use(A.GetNcols(), A.GetNrows(), C_mon); + return C; } - return Element; -} - -// ************************************* -/// @brief KS: Yaml emitter has problem and drops "", if you have special signs in you like * then there is problem. This bit hacky code adds these "" -/// @param yamlStr The YAML string to be processed (modified in-place). -inline void FixSampleNamesQuotes(std::string& yamlStr) { -// ************************************* - std::stringstream input(yamlStr); - std::string line; - std::string fixedYaml; - std::regex sampleNamesRegex(R"(SampleNames:\s*\[([^\]]+)\])"); - - while (std::getline(input, line)) { - std::smatch match; - if (std::regex_search(line, match, sampleNamesRegex)) { - std::string contents = match[1]; // inside the brackets - std::stringstream ss(contents); - std::string item; - std::vector quotedItems; - - while (std::getline(ss, item, ',')) { - item = std::regex_replace(item, std::regex(R"(^\s+|\s+$)"), ""); // trim - quotedItems.push_back("\"" + item + "\""); - } - std::string replacement = "SampleNames: [" + fmt::format("{}", fmt::join(quotedItems, ", ")) + "]"; - line = std::regex_replace(line, sampleNamesRegex, replacement); + // ******************************************** + /// @brief KS: Custom function to perform multiplication of matrix and vector + /// with multithreading + /// @param VecMulti Output Vector, VecMulti = matrix x vector + /// @param matrix This matrix is used for multiplication VecMulti = matrix x + /// vector + /// @param vector This vector is used for multiplication VecMulti = matrix x + /// vector + /// @param n this is size of matrix and vector, we assume matrix is symmetric + inline void MatrixVectorMulti(double *_restrict_ VecMulti, + double **_restrict_ matrix, + const double *_restrict_ vector, const int n) { + // ******************************************** +#ifdef MULTITHREAD +#pragma omp parallel for +#endif + for (int i = 0; i < n; ++i) { + double result = 0.0; +#ifdef MULTITHREAD +#pragma omp simd +#endif + for (int j = 0; j < n; ++j) { + result += matrix[i][j] * vector[j]; + } + VecMulti[i] = result; } - fixedYaml += line + "\n"; } - yamlStr = fixedYaml; -} - -// ************************************* -/// @brief KS: Add Tune values to YAML covariance matrix -/// @param root The root YAML node to be updated. -/// @param Values The values to add for the specified tune. -/// @param Tune The name of the tune (e.g., "PostFit"). -/// @param FancyNames Optional list of fancy names to match systematics (must match Values size if provided). -inline void AddTuneValues(YAML::Node& root, - const std::vector& Values, - const std::string& Tune, - const std::vector& FancyNames = {}) { -// ************************************* - YAML::Node NodeCopy = YAML::Clone(root); - YAML::Node systematics = NodeCopy["Systematics"]; - - if (!systematics || !systematics.IsSequence()) { - MACH3LOG_ERROR("'Systematics' node is missing or not a sequence in the YAML copy"); - throw MaCh3Exception(__FILE__, __LINE__); + // ******************************************** + /// @brief KS: Custom function to perform multiplication of matrix and single + /// element which is thread safe + /// @param matrix This matrix is used for multiplication VecMulti = matrix x + /// vector + /// @param vector This vector is used for multiplication VecMulti = matrix x + /// vector + /// @param Length this is size of matrix and vector, we assume matrix is + /// symmetric + /// @param i Element of matrix that we want to multiply + inline double MatrixVectorMultiSingle(double **_restrict_ matrix, + const double *_restrict_ vector, + const int Length, const int i) { + // ******************************************** + double Element = 0.0; +#ifdef MULTITHREAD +#pragma omp simd +#endif + for (int j = 0; j < Length; ++j) { + Element += matrix[i][j] * vector[j]; + } + return Element; } - if (!FancyNames.empty() && FancyNames.size() != Values.size()) { - MACH3LOG_ERROR("Mismatch in sizes: FancyNames has {}, but Values has {}", FancyNames.size(), Values.size()); - throw MaCh3Exception(__FILE__, __LINE__); - } + // ************************************* + /// @brief KS: Yaml emitter has problem and drops "", if you have special + /// signs in you like * then there is problem. This bit hacky code adds these + /// "" + /// @param yamlStr The YAML string to be processed (modified in-place). + inline void FixSampleNamesQuotes(std::string & yamlStr) { + // ************************************* + std::stringstream input(yamlStr); + std::string line; + std::string fixedYaml; + std::regex sampleNamesRegex(R"(SampleNames:\s*\[([^\]]+)\])"); + + while (std::getline(input, line)) { + std::smatch match; + if (std::regex_search(line, match, sampleNamesRegex)) { + std::string contents = match[1]; // inside the brackets + std::stringstream ss(contents); + std::string item; + std::vector quotedItems; + + while (std::getline(ss, item, ',')) { + item = + std::regex_replace(item, std::regex(R"(^\s+|\s+$)"), ""); // trim + quotedItems.push_back("\"" + item + "\""); + } - if (FancyNames.empty() && systematics.size() != Values.size()) { - MACH3LOG_ERROR("Mismatch in sizes: Values has {}, but YAML 'Systematics' has {} entries", - Values.size(), systematics.size()); - throw MaCh3Exception(__FILE__, __LINE__); + std::string replacement = + "SampleNames: [" + fmt::format("{}", fmt::join(quotedItems, ", ")) + + "]"; + line = std::regex_replace(line, sampleNamesRegex, replacement); + } + fixedYaml += line + "\n"; + } + + yamlStr = fixedYaml; } - if (!FancyNames.empty()) { - for (std::size_t i = 0; i < FancyNames.size(); ++i) { - bool matched = false; - for (std::size_t j = 0; j < systematics.size(); ++j) { - YAML::Node systematicNode = systematics[j]["Systematic"]; - if (!systematicNode) continue; - auto nameNode = systematicNode["Names"]; - if (!nameNode || !nameNode["FancyName"]) continue; - if (nameNode["FancyName"].as() == FancyNames[i]) { - if (!systematicNode["ParameterValues"]) { - MACH3LOG_ERROR("Missing 'ParameterValues' for matched FancyName '{}'", FancyNames[i]); - throw MaCh3Exception(__FILE__, __LINE__); + // ************************************* + /// @brief KS: Add Tune values to YAML covariance matrix + /// @param root The root YAML node to be updated. + /// @param Values The values to add for the specified tune. + /// @param Tune The name of the tune (e.g., "PostFit"). + /// @param FancyNames Optional list of fancy names to match systematics (must + /// match Values size if provided). + inline void AddTuneValues(YAML::Node & root, + const std::vector &Values, + const std::string &Tune, + const std::vector &FancyNames = {}) { + // ************************************* + YAML::Node NodeCopy = YAML::Clone(root); + YAML::Node systematics = NodeCopy["Systematics"]; + + if (!systematics || !systematics.IsSequence()) { + MACH3LOG_ERROR( + "'Systematics' node is missing or not a sequence in the YAML copy"); + throw MaCh3Exception(__FILE__, __LINE__); + } + + if (!FancyNames.empty() && FancyNames.size() != Values.size()) { + MACH3LOG_ERROR("Mismatch in sizes: FancyNames has {}, but Values has {}", + FancyNames.size(), Values.size()); + throw MaCh3Exception(__FILE__, __LINE__); + } + + if (FancyNames.empty() && systematics.size() != Values.size()) { + MACH3LOG_ERROR("Mismatch in sizes: Values has {}, but YAML 'Systematics' " + "has {} entries", + Values.size(), systematics.size()); + throw MaCh3Exception(__FILE__, __LINE__); + } + + if (!FancyNames.empty()) { + for (std::size_t i = 0; i < FancyNames.size(); ++i) { + bool matched = false; + for (std::size_t j = 0; j < systematics.size(); ++j) { + YAML::Node systematicNode = systematics[j]["Systematic"]; + if (!systematicNode) + continue; + auto nameNode = systematicNode["Names"]; + if (!nameNode || !nameNode["FancyName"]) + continue; + if (nameNode["FancyName"].as() == FancyNames[i]) { + if (!systematicNode["ParameterValues"]) { + MACH3LOG_ERROR( + "Missing 'ParameterValues' for matched FancyName '{}'", + FancyNames[i]); + throw MaCh3Exception(__FILE__, __LINE__); + } + systematicNode["ParameterValues"][Tune] = + MaCh3Utils::FormatDouble(Values[i], 4); + matched = true; + break; } - systematicNode["ParameterValues"][Tune] = MaCh3Utils::FormatDouble(Values[i], 4); - matched = true; - break; + } + if (!matched) { + MACH3LOG_ERROR( + "Could not find a matching FancyName '{}' in the systematics", + FancyNames[i]); + throw MaCh3Exception(__FILE__, __LINE__); } } - if (!matched) { - MACH3LOG_ERROR("Could not find a matching FancyName '{}' in the systematics", FancyNames[i]); - throw MaCh3Exception(__FILE__, __LINE__); + } else { + for (std::size_t i = 0; i < systematics.size(); ++i) { + YAML::Node systematicNode = systematics[i]["Systematic"]; + if (!systematicNode || !systematicNode["ParameterValues"]) { + MACH3LOG_ERROR( + "Missing 'Systematic' or 'ParameterValues' entry at index {}", i); + throw MaCh3Exception(__FILE__, __LINE__); + } + systematicNode["ParameterValues"][Tune] = + MaCh3Utils::FormatDouble(Values[i], 4); } } - } else { - for (std::size_t i = 0; i < systematics.size(); ++i) { - YAML::Node systematicNode = systematics[i]["Systematic"]; - if (!systematicNode || !systematicNode["ParameterValues"]) { - MACH3LOG_ERROR("Missing 'Systematic' or 'ParameterValues' entry at index {}", i); + + // Convert updated copy to string + std::string YAMLString = YAMLtoSTRING(NodeCopy); + FixSampleNamesQuotes(YAMLString); + // Write to output file + std::string OutName = "UpdatedMatrixWithTune" + Tune + ".yaml"; + std::ofstream outFile(OutName); + if (!outFile) { + MACH3LOG_ERROR("Failed to open file for writing: {}", OutName); + throw MaCh3Exception(__FILE__, __LINE__); + } + + outFile << YAMLString; + outFile.close(); + } + + // ************************************* + /// @brief KS: Replace correlation matrix and tune values in YAML covariance + /// matrix + /// @param root The root YAML node to be updated. + /// @param Values The new values for each systematic. + /// @param Errors The new errors for each systematic. + /// @param Correlation The new correlation matrix (must be square and match + /// Values size). + /// @param OutYAMLName The output filename for the updated YAML. + /// @param FancyNames Optional list of fancy names to match systematics (must + /// match Values size if provided). + inline void MakeCorrelationMatrix( + YAML::Node & root, const std::vector &Values, + const std::vector &Errors, + const std::vector> &Correlation, + const std::string &OutYAMLName, + const std::vector &FancyNames = {}) { + // ************************************* + if (Values.size() != Errors.size() || Values.size() != Correlation.size()) { + MACH3LOG_ERROR( + "Size mismatch between Values, Errors, and Correlation matrix"); + throw MaCh3Exception(__FILE__, __LINE__); + } + + for (const auto &row : Correlation) { + if (row.size() != Correlation.size()) { + MACH3LOG_ERROR("Correlation matrix is not square"); throw MaCh3Exception(__FILE__, __LINE__); } - systematicNode["ParameterValues"][Tune] = MaCh3Utils::FormatDouble(Values[i], 4); } - } - // Convert updated copy to string - std::string YAMLString = YAMLtoSTRING(NodeCopy); - FixSampleNamesQuotes(YAMLString); - // Write to output file - std::string OutName = "UpdatedMatrixWithTune" + Tune + ".yaml"; - std::ofstream outFile(OutName); - if (!outFile) { - MACH3LOG_ERROR("Failed to open file for writing: {}", OutName); - throw MaCh3Exception(__FILE__, __LINE__); - } + YAML::Node NodeCopy = YAML::Clone(root); + YAML::Node systematics = NodeCopy["Systematics"]; - outFile << YAMLString; - outFile.close(); -} - -// ************************************* -/// @brief KS: Replace correlation matrix and tune values in YAML covariance matrix -/// @param root The root YAML node to be updated. -/// @param Values The new values for each systematic. -/// @param Errors The new errors for each systematic. -/// @param Correlation The new correlation matrix (must be square and match Values size). -/// @param OutYAMLName The output filename for the updated YAML. -/// @param FancyNames Optional list of fancy names to match systematics (must match Values size if provided). -inline void MakeCorrelationMatrix(YAML::Node& root, - const std::vector& Values, - const std::vector& Errors, - const std::vector>& Correlation, - const std::string& OutYAMLName, - const std::vector& FancyNames = {}) { -// ************************************* - if (Values.size() != Errors.size() || Values.size() != Correlation.size()) { - MACH3LOG_ERROR("Size mismatch between Values, Errors, and Correlation matrix"); - throw MaCh3Exception(__FILE__, __LINE__); - } + if (!systematics || !systematics.IsSequence()) { + MACH3LOG_ERROR("'Systematics' node is missing or not a sequence"); + throw MaCh3Exception(__FILE__, __LINE__); + } - for (const auto& row : Correlation) { - if (row.size() != Correlation.size()) { - MACH3LOG_ERROR("Correlation matrix is not square"); + if (!FancyNames.empty() && FancyNames.size() != Values.size()) { + MACH3LOG_ERROR("FancyNames size ({}) does not match Values size ({})", + FancyNames.size(), Values.size()); throw MaCh3Exception(__FILE__, __LINE__); } - } - YAML::Node NodeCopy = YAML::Clone(root); - YAML::Node systematics = NodeCopy["Systematics"]; + // Map from FancyName to Systematic node + std::unordered_map nameToNode; + for (std::size_t i = 0; i < systematics.size(); ++i) { + YAML::Node syst = systematics[i]["Systematic"]; + if (!syst || !syst["Names"] || !syst["Names"]["FancyName"]) + continue; + std::string name = syst["Names"]["FancyName"].as(); + nameToNode[name] = syst; + } - if (!systematics || !systematics.IsSequence()) { - MACH3LOG_ERROR("'Systematics' node is missing or not a sequence"); - throw MaCh3Exception(__FILE__, __LINE__); - } + if (!FancyNames.empty()) { + for (std::size_t i = 0; i < FancyNames.size(); ++i) { + const std::string &name_i = FancyNames[i]; + auto it_i = nameToNode.find(name_i); + if (it_i == nameToNode.end()) { + MACH3LOG_ERROR("Could not find FancyName '{}' in YAML", name_i); + throw MaCh3Exception(__FILE__, __LINE__); + } + YAML::Node &syst_i = it_i->second; + + syst_i["ParameterValues"]["PreFitValue"] = + MaCh3Utils::FormatDouble(Values[i], 4); + syst_i["Error"] = std::round(Errors[i] * 100.0) / 100.0; + + YAML::Node correlationsNode = YAML::Node(YAML::NodeType::Sequence); + for (std::size_t j = 0; j < FancyNames.size(); ++j) { + if (i == j) + continue; + // KS: Skip if value close to 0 + if (std::abs(Correlation[i][j]) < 1e-8) + continue; + YAML::Node singleEntry; + singleEntry[FancyNames[j]] = + MaCh3Utils::FormatDouble(Correlation[i][j], 4); + correlationsNode.push_back(singleEntry); + } + syst_i["Correlations"] = correlationsNode; + } + } else { + if (systematics.size() != Values.size()) { + MACH3LOG_ERROR("Mismatch in sizes: Values has {}, but YAML " + "'Systematics' has {} entries", + Values.size(), systematics.size()); + throw MaCh3Exception(__FILE__, __LINE__); + } - if (!FancyNames.empty() && FancyNames.size() != Values.size()) { - MACH3LOG_ERROR("FancyNames size ({}) does not match Values size ({})", FancyNames.size(), Values.size()); - throw MaCh3Exception(__FILE__, __LINE__); - } + for (std::size_t i = 0; i < systematics.size(); ++i) { + YAML::Node syst = systematics[i]["Systematic"]; + if (!syst) { + MACH3LOG_ERROR("Missing 'Systematic' node at index {}", i); + throw MaCh3Exception(__FILE__, __LINE__); + } + + syst["ParameterValues"]["PreFitValue"] = + MaCh3Utils::FormatDouble(Values[i], 4); + syst["Error"] = std::round(Errors[i] * 100.0) / 100.0; + + YAML::Node correlationsNode = YAML::Node(YAML::NodeType::Sequence); + for (std::size_t j = 0; j < Correlation[i].size(); ++j) { + if (i == j) + continue; + // KS: Skip if value close to 0 + if (std::abs(Correlation[i][j]) < 1e-8) + continue; + YAML::Node singleEntry; + const std::string &otherName = + systematics[j]["Systematic"]["Names"]["FancyName"] + .as(); + singleEntry[otherName] = + MaCh3Utils::FormatDouble(Correlation[i][j], 4); + correlationsNode.push_back(singleEntry); + } + syst["Correlations"] = correlationsNode; + } + } - // Map from FancyName to Systematic node - std::unordered_map nameToNode; - for (std::size_t i = 0; i < systematics.size(); ++i) { - YAML::Node syst = systematics[i]["Systematic"]; - if (!syst || !syst["Names"] || !syst["Names"]["FancyName"]) continue; - std::string name = syst["Names"]["FancyName"].as(); - nameToNode[name] = syst; + // Convert and write + std::string YAMLString = YAMLtoSTRING(NodeCopy); + FixSampleNamesQuotes(YAMLString); + std::ofstream outFile(OutYAMLName); + if (!outFile) { + MACH3LOG_ERROR("Failed to open file for writing: {}", OutYAMLName); + throw MaCh3Exception(__FILE__, __LINE__); + } + + outFile << YAMLString; + outFile.close(); } - if (!FancyNames.empty()) { - for (std::size_t i = 0; i < FancyNames.size(); ++i) { - const std::string& name_i = FancyNames[i]; - auto it_i = nameToNode.find(name_i); - if (it_i == nameToNode.end()) { - MACH3LOG_ERROR("Could not find FancyName '{}' in YAML", name_i); - throw MaCh3Exception(__FILE__, __LINE__); + // ************************************* + /// @brief KS: We store configuration macros inside the chain. + /// In the past, multiple configs were stored, which required error-prone + /// hardcoding like "Config_xsec_cov". Therefore, this code maintains backward + /// compatibility by checking the number of macros present and using a + /// hardcoded name only if necessary. + inline TMacro *GetConfigMacroFromChain(TDirectory * CovarianceFolder) { + // ************************************* + if (!CovarianceFolder) { + MACH3LOG_ERROR("Null TDirectory passed to {}", __func__); + throw MaCh3Exception(__FILE__, __LINE__); + } + + TMacro *foundMacro = nullptr; + int macroCount = 0; + + TIter next(CovarianceFolder->GetListOfKeys()); + TKey *key; + while ((key = dynamic_cast(next()))) { + if (std::string(key->GetClassName()) == "TMacro") { + ++macroCount; + if (macroCount == 1) { + foundMacro = dynamic_cast(key->ReadObj()); + } } - YAML::Node& syst_i = it_i->second; - - syst_i["ParameterValues"]["PreFitValue"] = MaCh3Utils::FormatDouble(Values[i], 4); - syst_i["Error"] = std::round(Errors[i] * 100.0) / 100.0; - - YAML::Node correlationsNode = YAML::Node(YAML::NodeType::Sequence); - for (std::size_t j = 0; j < FancyNames.size(); ++j) { - if (i == j) continue; - // KS: Skip if value close to 0 - if (std::abs(Correlation[i][j]) < 1e-8) continue; - YAML::Node singleEntry; - singleEntry[FancyNames[j]] = MaCh3Utils::FormatDouble(Correlation[i][j], 4); - correlationsNode.push_back(singleEntry); + } + + if (macroCount == 1 && foundMacro) { + MACH3LOG_INFO("Found single TMacro in directory: using it."); + return foundMacro; + } else { + MACH3LOG_WARN("Found {} TMacro objects. Using hardcoded macro name: " + "Config_xsec_cov.", + macroCount); + TMacro *fallback = CovarianceFolder->Get("Config_xsec_cov"); + if (!fallback) { + MACH3LOG_WARN( + "Fallback macro 'Config_xsec_cov' not found in directory."); } - syst_i["Correlations"] = correlationsNode; + return fallback; } - } else { - if (systematics.size() != Values.size()) { - MACH3LOG_ERROR("Mismatch in sizes: Values has {}, but YAML 'Systematics' has {} entries", - Values.size(), systematics.size()); + } + + // ************************************* + /// @brief KS: Retrieve the cross-section covariance matrix from the given + /// TDirectory. Historically, multiple covariance matrices could be stored, + /// requiring fragile hardcoded paths like "CovarianceFolder/xsec_cov". This + /// function maintains backward compatibility by: + /// - Using the single matrix present if only one exists, + /// - Otherwise falling back to the hardcoded path. + /// This avoids error-prone assumptions while supporting both old and new + /// formats. + inline TMatrixDSym *GetCovMatrixFromChain(TDirectory * TempFile) { + // ************************************* + if (!TempFile) { + MACH3LOG_ERROR("Null TDirectory passed to {}.", __func__); throw MaCh3Exception(__FILE__, __LINE__); } - for (std::size_t i = 0; i < systematics.size(); ++i) { - YAML::Node syst = systematics[i]["Systematic"]; - if (!syst) { - MACH3LOG_ERROR("Missing 'Systematic' node at index {}", i); - throw MaCh3Exception(__FILE__, __LINE__); + TMatrixDSym *foundMatrix = nullptr; + int matrixCount = 0; + + TIter next(TempFile->GetListOfKeys()); + TKey *key; + while ((key = dynamic_cast(next()))) { + std::string className = key->GetClassName(); + if (className.find("TMatrix") != std::string::npos) { + ++matrixCount; + if (matrixCount == 1) { + foundMatrix = dynamic_cast(key->ReadObj()); + } } + } - syst["ParameterValues"]["PreFitValue"] = MaCh3Utils::FormatDouble(Values[i], 4); - syst["Error"] = std::round(Errors[i] * 100.0) / 100.0; - - YAML::Node correlationsNode = YAML::Node(YAML::NodeType::Sequence); - for (std::size_t j = 0; j < Correlation[i].size(); ++j) { - if (i == j) continue; - // KS: Skip if value close to 0 - if (std::abs(Correlation[i][j]) < 1e-8) continue; - YAML::Node singleEntry; - const std::string& otherName = systematics[j]["Systematic"]["Names"]["FancyName"].as(); - singleEntry[otherName] = MaCh3Utils::FormatDouble(Correlation[i][j], 4); - correlationsNode.push_back(singleEntry); + if (matrixCount == 1 && foundMatrix) { + MACH3LOG_INFO("Found single TMatrixDSym in directory: using it."); + return foundMatrix; + } else { + MACH3LOG_WARN( + "Found {} TMatrixDSym objects. Using hardcoded path: xsec_cov.", + matrixCount); + TMatrixDSym *fallback = TempFile->Get("xsec_cov"); + if (!fallback) { + MACH3LOG_WARN("Fallback matrix 'xsec_cov' not found."); } - syst["Correlations"] = correlationsNode; + return fallback; } } - // Convert and write - std::string YAMLString = YAMLtoSTRING(NodeCopy); - FixSampleNamesQuotes(YAMLString); - std::ofstream outFile(OutYAMLName); - if (!outFile) { - MACH3LOG_ERROR("Failed to open file for writing: {}", OutYAMLName); - throw MaCh3Exception(__FILE__, __LINE__); + // ************************************* + /// @brief Computes Cholesky decomposition of a symmetric positive definite + /// matrix using custom function which can be even 20 times faster + /// @param matrix Input symmetric positive definite matrix + /// @param matrixName Identifier for error reporting + inline std::vector> GetCholeskyDecomposedMatrix( + const TMatrixDSym &matrix, const std::string &matrixName) { + // ************************************* + const Int_t n = matrix.GetNrows(); + std::vector> L(n, std::vector(n, 0.0)); + + for (Int_t j = 0; j < n; ++j) { + // Compute diagonal element (must be serial) + double sum_diag = matrix(j, j); + for (Int_t k = 0; k < j; ++k) { + sum_diag -= L[j][k] * L[j][k]; + } + const double tol = 1e-15; + if (sum_diag <= tol) { + MACH3LOG_ERROR( + "Cholesky decomposition failed for {} (non-positive diagonal)", + matrixName); + throw MaCh3Exception(__FILE__, __LINE__); + } + L[j][j] = std::sqrt(sum_diag); + +// Compute the rest of the column in parallel +#ifdef MULTITHREAD +#pragma omp parallel for +#endif + for (Int_t i = j + 1; i < n; ++i) { + double sum = matrix(i, j); + for (Int_t k = 0; k < j; ++k) { + sum -= L[i][k] * L[j][k]; + } + L[i][j] = sum / L[j][j]; + } + } + return L; } - outFile << YAMLString; - outFile.close(); -} - -// ************************************* -/// @brief KS: We store configuration macros inside the chain. -/// In the past, multiple configs were stored, which required error-prone hardcoding like "Config_xsec_cov". -/// Therefore, this code maintains backward compatibility by checking the number of macros present and -/// using a hardcoded name only if necessary. -inline TMacro* GetConfigMacroFromChain(TDirectory* CovarianceFolder) { -// ************************************* - if (!CovarianceFolder) { - MACH3LOG_ERROR("Null TDirectory passed to {}", __func__); - throw MaCh3Exception(__FILE__, __LINE__); + // ************************************* + /// @brief Checks if a matrix can be Cholesky decomposed + /// @param matrix Input symmetric matrix to test + inline bool CanDecomposeMatrix(const TMatrixDSym &matrix) { + // ************************************* + TDecompChol chdcmp(matrix); + return chdcmp.Decompose(); } - TMacro* foundMacro = nullptr; - int macroCount = 0; - - TIter next(CovarianceFolder->GetListOfKeys()); - TKey* key; - while ((key = dynamic_cast(next()))) { - if (std::string(key->GetClassName()) == "TMacro") { - ++macroCount; - if (macroCount == 1) { - foundMacro = dynamic_cast(key->ReadObj()); + // ************************************* + /// @brief Makes sure that matrix is positive-definite by adding a small + /// number to on-diagonal elements + inline void MakeMatrixPosDef(TMatrixDSym * cov) { + // ************************************* + // DB Save original warning state and then increase it in this function to + // suppress 'matrix not positive definite' messages Means we no longer need + // to overload + int originalErrorWarning = gErrorIgnoreLevel; + gErrorIgnoreLevel = kFatal; + + // DB Loop 1000 times adding 1e-9 which tops out at 1e-6 shift on the + // diagonal before throwing error + constexpr int MaxAttempts = 1e5; + const int matrixSize = cov->GetNrows(); + int iAttempt = 0; + bool CanDecomp = false; + + for (iAttempt = 0; iAttempt < MaxAttempts; iAttempt++) { + if (CanDecomposeMatrix(*cov)) { + CanDecomp = true; + break; + } else { +#ifdef MULTITHREAD +#pragma omp parallel for +#endif + for (int iVar = 0; iVar < matrixSize; iVar++) { + (*cov)(iVar, iVar) += pow(10, -9); + } } } - } - if (macroCount == 1 && foundMacro) { - MACH3LOG_INFO("Found single TMacro in directory: using it."); - return foundMacro; - } else { - MACH3LOG_WARN("Found {} TMacro objects. Using hardcoded macro name: Config_xsec_cov.", macroCount); - TMacro* fallback = CovarianceFolder->Get("Config_xsec_cov"); - if (!fallback) { - MACH3LOG_WARN("Fallback macro 'Config_xsec_cov' not found in directory."); + if (!CanDecomp) { + MACH3LOG_ERROR("Tried {} times to shift diagonal but still can not " + "decompose the matrix", + MaxAttempts); + MACH3LOG_ERROR( + "This indicates that something is wrong with the input matrix"); + throw MaCh3Exception(__FILE__, __LINE__); } - return fallback; - } -} - -// ************************************* -/// @brief KS: Retrieve the cross-section covariance matrix from the given TDirectory. -/// Historically, multiple covariance matrices could be stored, requiring fragile hardcoded paths like "CovarianceFolder/xsec_cov". -/// This function maintains backward compatibility by: -/// - Using the single matrix present if only one exists, -/// - Otherwise falling back to the hardcoded path. -/// This avoids error-prone assumptions while supporting both old and new formats. -inline TMatrixDSym* GetCovMatrixFromChain(TDirectory* TempFile) { -// ************************************* - if (!TempFile) { - MACH3LOG_ERROR("Null TDirectory passed to {}.", __func__); - throw MaCh3Exception(__FILE__, __LINE__); + + // DB Resetting warning level + gErrorIgnoreLevel = originalErrorWarning; } - TMatrixDSym* foundMatrix = nullptr; - int matrixCount = 0; - - TIter next(TempFile->GetListOfKeys()); - TKey* key; - while ((key = dynamic_cast(next()))) { - std::string className = key->GetClassName(); - if (className.find("TMatrix") != std::string::npos) { - ++matrixCount; - if (matrixCount == 1) { - foundMatrix = dynamic_cast(key->ReadObj()); + // ************************************* + /// @brief Makes sure that matrix is positive-definite by adding a small + /// number to on-diagonal elements + inline Eigen::MatrixXd MakeMatrixPosDef(Eigen::MatrixXd cov) { + // ************************************* + + // DB Loop 1000 times adding 1e-9 which tops out at 1e-6 shift on the + // diagonal before throwing error + constexpr int MaxAttempts = 1e5; + + for (int iAttempt = 0; iAttempt < MaxAttempts; iAttempt++) { + if (Eigen::LLT(cov).info() == Eigen::Success) { + return cov; + } else { + cov.diagonal() += 1E-9; } } + + MACH3LOG_ERROR("Tried {} times to shift diagonal but still can not " + "decompose the matrix", + MaxAttempts); + MACH3LOG_ERROR( + "This indicates that something is wrong with the input matrix"); + throw MaCh3Exception(__FILE__, __LINE__); } - if (matrixCount == 1 && foundMatrix) { - MACH3LOG_INFO("Found single TMatrixDSym in directory: using it."); - return foundMatrix; - } else { - MACH3LOG_WARN("Found {} TMatrixDSym objects. Using hardcoded path: xsec_cov.", matrixCount); - TMatrixDSym* fallback = TempFile->Get("xsec_cov"); - if (!fallback) { - MACH3LOG_WARN("Fallback matrix 'xsec_cov' not found."); + // ******************************************** + // HW: Finds closest possible positive definite matrix in Frobenius Norm + // ||.||_frob + // Where ||X||_frob=sqrt[sum_ij(x_ij^2)] (basically just turns an n,n matrix + // into vector in n^2 space then does Euclidean norm) + void MakeClosestPosDef(TMatrixDSym * cov) { + // ******************************************** + // Want to get cov' = (cov_sym+cov_polar)/2 + // cov_sym=(cov+cov^T)/2 + // cov_polar-> SVD cov to cov=USV^T then cov_polar=VSV^T + + // Get frob norm of cov + // Double_t cov_norm=cov->E2Norm(); + + TMatrixDSym *cov_trans = cov; + cov_trans->T(); + TMatrixDSym cov_sym = + 0.5 * (*cov + *cov_trans); // If cov is symmetric does nothing, + // otherwise just ensures symmetry + + // Do SVD to get polar form + TDecompSVD cov_sym_svd = TDecompSVD(cov_sym); + if (!cov_sym_svd.Decompose()) { + MACH3LOG_WARN( + "Cannot do SVD on input matrix, trying MakePosDef() first!"); + MakePosDef(&cov_sym); } - return fallback; + + TMatrixD cov_sym_v = cov_sym_svd.GetV(); + TMatrixD cov_sym_vt = cov_sym_v; + cov_sym_vt.T(); + // SVD returns as vector (grrr) so need to get into matrix form for + // multiplying! + TVectorD cov_sym_sigvect = cov_sym_svd.GetSig(); + + const Int_t nCols = + cov_sym_v.GetNcols(); // square so only need rows hence lack of cols + TMatrixDSym cov_sym_sig(nCols); + TMatrixDDiag cov_sym_sig_diag(cov_sym_sig); + cov_sym_sig_diag = cov_sym_sigvect; + + // Can finally get H=VSV + TMatrixDSym cov_sym_polar = cov_sym_sig.SimilarityT( + cov_sym_vt); // V*S*V^T (this took forver to find!) + + // Now we can construct closest approximater Ahat=0.5*(B+H) + TMatrixDSym cov_closest_approx = + 0.5 * (cov_sym + cov_sym_polar); // Not fully sure why this is even + // needed since symmetric B -> U=V + // Get norm of transformed + // Double_t approx_norm=cov_closest_approx.E2Norm(); + // MACH3LOG_INFO("Initial Norm: {:.6f} | Norm after transformation: {:.6f} | + // Ratio: {:.6f}", cov_norm, approx_norm, cov_norm / approx_norm); + + *cov = cov_closest_approx; + // Now can just add a makeposdef! + MakePosDef(cov); } -} - -// ************************************* -/// @brief Computes Cholesky decomposition of a symmetric positive definite matrix using custom function which can be even 20 times faster -/// @param matrix Input symmetric positive definite matrix -/// @param matrixName Identifier for error reporting -inline std::vector> GetCholeskyDecomposedMatrix(const TMatrixDSym& matrix, const std::string& matrixName) { -// ************************************* - const Int_t n = matrix.GetNrows(); - std::vector> L(n, std::vector(n, 0.0)); - - for (Int_t j = 0; j < n; ++j) { - // Compute diagonal element (must be serial) - double sum_diag = matrix(j, j); - for (Int_t k = 0; k < j; ++k) { - sum_diag -= L[j][k] * L[j][k]; - } - const double tol = 1e-15; - if (sum_diag <= tol) { - MACH3LOG_ERROR("Cholesky decomposition failed for {} (non-positive diagonal)", matrixName); - throw MaCh3Exception(__FILE__, __LINE__); + + // ******************************************** + // KS: Convert covariance matrix to correlation matrix and return TH2D which + // can be used for fancy plotting + TH2D *GetCorrelationMatrix(Eigen::MatrixXd const &covariance) { + // ******************************************** + TH2D *hMatrix = + new TH2D(GetName().c_str(), GetName().c_str(), covariance.rows(), 0.0, + covariance.rows(), covariance.rows(), 0.0, covariance.rows()); + hMatrix->SetDirectory(nullptr); + for (int i = 0; i < covariance.rows(); i++) { + hMatrix->SetBinContent(i + 1, i + 1, 1.); + hMatrix->GetXaxis()->SetBinLabel(i + 1, GetParFancyName(i).c_str()); + hMatrix->GetYaxis()->SetBinLabel(i + 1, GetParFancyName(i).c_str()); } - L[j][j] = std::sqrt(sum_diag); - // Compute the rest of the column in parallel - #ifdef MULTITHREAD - #pragma omp parallel for - #endif - for (Int_t i = j + 1; i < n; ++i) { - double sum = matrix(i, j); - for (Int_t k = 0; k < j; ++k) { - sum -= L[i][k] * L[j][k]; + auto sqrt_diag = covariance.diagonal().cwiseSqrt(); + + for (int i = 0; i < covariance.rows(); i++) { + for (int j = 0; j <= i; j++) { + const double Corr = covariance(i, j) / sqrt_diag(i) * sqrt_diag(j); + hMatrix->SetBinContent(i + 1, j + 1, Corr); + hMatrix->SetBinContent(j + 1, i + 1, Corr); } - L[i][j] = sum / L[j][j]; } + return hMatrix; } - return L; -} - -// ************************************* -/// @brief Checks if a matrix can be Cholesky decomposed -/// @param matrix Input symmetric matrix to test -inline bool CanDecomposeMatrix(const TMatrixDSym& matrix) { -// ************************************* - TDecompChol chdcmp(matrix); - return chdcmp.Decompose(); -} - -// ************************************* -/// @brief Makes sure that matrix is positive-definite by adding a small number to on-diagonal elements -inline void MakeMatrixPosDef(TMatrixDSym *cov) { -// ************************************* - //DB Save original warning state and then increase it in this function to suppress 'matrix not positive definite' messages - //Means we no longer need to overload - int originalErrorWarning = gErrorIgnoreLevel; - gErrorIgnoreLevel = kFatal; - - //DB Loop 1000 times adding 1e-9 which tops out at 1e-6 shift on the diagonal before throwing error - constexpr int MaxAttempts = 1e5; - const int matrixSize = cov->GetNrows(); - int iAttempt = 0; - bool CanDecomp = false; - - for (iAttempt = 0; iAttempt < MaxAttempts; iAttempt++) { - if (CanDecomposeMatrix(*cov)) { - CanDecomp = true; - break; - } else { - #ifdef MULTITHREAD - #pragma omp parallel for - #endif - for (int iVar = 0 ; iVar < matrixSize; iVar++) { - (*cov)(iVar, iVar) += 1e-9; + + Eigen::MatrixXd ROOTToEigen(TMatrixDSym const &mat) { + auto emat = Eigen::MatrixXd(mat.GetNrows(), mat.GetNcols()); + for (int ri = 0; ri < mat.GetNrows(); ++ri) { + for (int ci = 0; ci < mat.GetNcols(); ++ci) { + emat(ri, ci) = mat(ri, ci); } } + return emat; } - if (!CanDecomp) { - MACH3LOG_ERROR("Tried {} times to shift diagonal but still can not decompose the matrix", MaxAttempts); - MACH3LOG_ERROR("This indicates that something is wrong with the input matrix"); - throw MaCh3Exception(__FILE__ , __LINE__ ); + void EigenToROOT(Eigen::MatrixXd const &emat, TMatrixDSym &mat) { + mat.ResizeTo(emat.rows(), emat.cols()); + for (int ri = 0; ri < emat.rows(); ++ri) { + for (int ci = 0; ci < emat.cols(); ++ci) { + mat(ri, ci) = emat(ri, ci); + } + } } - //DB Resetting warning level - gErrorIgnoreLevel = originalErrorWarning; -} + std::vector EigenToStdVector(Eigen::VectorXd const &evec) { + return std::vector(evec.begin(), evec.end()); + } + + Eigen::VectorXd StdVectorToEigen(std::vector const &svec) { + Eigen::VectorXd evec(svec.size()); + for (size_t i = 0; i < svec; ++i) { + evec(i) = svec[i]; + } + return evec; + } + + void EnsureNoOutOfBlockCorrelations(Eigen::MatrixXd const &CovMatrix, + std::array block_definition, + double threshold = 1E-6) { + + for (int i = block_definition[0]; i <= block_definition[1]; ++i) { + for (int j = 0; j < CovMatrix.rows(); ++j) { + // Skip if j is inside the decomposed range (we only want + // cross-correlations) + if ((j >= block_definition[0]) && (j <= block_definition[1])) { + continue; + } + + if (std::fabs(CovMatrix(i, j)) > correlation_threshold) { + found_significant_correlation = true; + MACH3LOG_ERROR("Significant correlation detected between decomposed " + "parameter '{}' " + "and undecomposed parameter '{}': {:.6e}", + i, j, CovMatrix(i, j)); + throw MaCh3Exception(__FILE__, __LINE__); + } + } + } + } } // end M3 namespace diff --git a/Parameters/ParameterTunes.h b/Parameters/ParameterTunes.h index 89c783354..b288b4192 100644 --- a/Parameters/ParameterTunes.h +++ b/Parameters/ParameterTunes.h @@ -9,6 +9,7 @@ class ParameterTunes{ public: /// @brief Constructor + ParameterTunes(){}; ParameterTunes(const YAML::Node& Settings); /// @brief virtual ~ParameterTunes(); @@ -20,8 +21,8 @@ class ParameterTunes{ /// @brief Simply print all tunes and associated values void PrintTunes() const; private: - /// Name of each Tun - std::vector TuneNames; + /// Name of each Tune + std::vector TuneNames; /// Values for each Tune and Parameter std::vector> TuneValues; /// Map between tune name and value diff --git a/cmake/Modules/MaCh3Dependencies.cmake b/cmake/Modules/MaCh3Dependencies.cmake index 181d282fc..c71da4b30 100755 --- a/cmake/Modules/MaCh3Dependencies.cmake +++ b/cmake/Modules/MaCh3Dependencies.cmake @@ -114,3 +114,11 @@ if( MaCh3_PYTHON_ENABLED ) GIT_TAG v2.13.5 ) endif() + +FetchContent_Declare( + Eigen3 + GIT_REPOSITORY https://gitlab.com/libeigen/eigen.git + GIT_TAG 5.0.1 +) +SET(EIGEN_BUILD_CMAKE_PACKAGE ON) #export targets +FetchContent_MakeAvailable(Eigen3) From 5e0ec65be0800031cf458cd1577016081e47ae03 Mon Sep 17 00:00:00 2001 From: Luke Pickering Date: Tue, 16 Dec 2025 17:06:15 +0000 Subject: [PATCH 02/14] Continues redesign. Split up ParameterHandlerBase functionality. - Adds ParameterList and StepProposer classes - Adds README that will contain design decisions - Removes PCAhandler as a separate class - Adds some local tests --- Manager/Core.h | 3 + Parameters/CMakeLists.txt | 9 +- Parameters/PCA.h | 69 +++ Parameters/PCAHandler.cpp | 729 -------------------------- Parameters/PCAHandler.h | 262 --------- Parameters/ParameterHandlerBase.cpp | 630 +++++++++------------- Parameters/ParameterHandlerBase.h | 135 +++-- Parameters/ParameterHandlerUtils.h | 63 ++- Parameters/ParameterList.cpp | 478 +++++++++++++++++ Parameters/ParameterList.h | 105 ++++ Parameters/README.md | 14 + Parameters/StepProposer.cpp | 66 +++ Parameters/StepProposer.h | 70 +++ Parameters/test/CMakeLists.txt | 19 + Parameters/test/ParameterListTest.cpp | 46 ++ Parameters/test/StepProposerTest.cpp | 1 + 16 files changed, 1243 insertions(+), 1456 deletions(-) create mode 100644 Parameters/PCA.h delete mode 100755 Parameters/PCAHandler.cpp delete mode 100644 Parameters/PCAHandler.h create mode 100644 Parameters/ParameterList.cpp create mode 100644 Parameters/ParameterList.h create mode 100644 Parameters/README.md create mode 100644 Parameters/StepProposer.cpp create mode 100644 Parameters/StepProposer.h create mode 100644 Parameters/test/CMakeLists.txt create mode 100644 Parameters/test/ParameterListTest.cpp create mode 100644 Parameters/test/StepProposerTest.cpp diff --git a/Manager/Core.h b/Manager/Core.h index 243d90a53..62fdb1da7 100755 --- a/Manager/Core.h +++ b/Manager/Core.h @@ -123,6 +123,9 @@ _Pragma("GCC diagnostic ignored \"-Wswitch-enum\"") \ _Pragma("GCC diagnostic ignored \"-Wconversion\"") \ _Pragma("GCC diagnostic ignored \"-Wshadow\"") \ _Pragma("GCC diagnostic ignored \"-Wswitch-enum\"") +_Pragma("GCC diagnostic ignored \"-Wduplicated-branches\"") +_Pragma("GCC diagnostic ignored \"-Wstrict-aliasing\"") + /// @brief KS: Restore warning checking after including external headers #define _MaCh3_Safe_Include_End_ \ _Pragma("GCC diagnostic pop") diff --git a/Parameters/CMakeLists.txt b/Parameters/CMakeLists.txt index a5bfc2928..5484e97ad 100644 --- a/Parameters/CMakeLists.txt +++ b/Parameters/CMakeLists.txt @@ -9,11 +9,11 @@ set(HEADERS ) add_library(Parameters SHARED - ParameterHandlerBase.cpp - ParameterHandlerGeneric.cpp - AdaptiveMCMCHandler.cpp - PCAHandler.cpp + # ParameterHandlerBase.cpp + # ParameterHandlerGeneric.cpp ParameterTunes.cpp + StepProposer.cpp + ParameterList.cpp ) set_target_properties(Parameters PROPERTIES @@ -51,3 +51,4 @@ install(TARGETS Parameters add_library(MaCh3::Parameters ALIAS Parameters) +add_subdirectory(test) diff --git a/Parameters/PCA.h b/Parameters/PCA.h new file mode 100644 index 000000000..c23a1ee6a --- /dev/null +++ b/Parameters/PCA.h @@ -0,0 +1,69 @@ +#pragma once + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Walloca" +#pragma GCC diagnostic ignored "-Wold-style-cast" +#pragma GCC diagnostic ignored "-Wuseless-cast" +#pragma GCC diagnostic ignored "-Wconversion" +#pragma GCC diagnostic ignored "-Wduplicated-branches" +#pragma GCC diagnostic ignored "-Wstrict-aliasing" +#include "Eigen/Dense" +#pragma GCC diagnostic pop + +#include +#include + +Eigen::MatrixXd CalculateTruncatedPCARotation(Eigen::MatrixXd mx_phyiscs_basis, + double threshold) { + + // a covariance is real symmetric, so self adjoint + Eigen::SelfAdjointEigenSolver EigenSolver(mx_phyiscs_basis); + + if (EigenSolver.info() != Eigen::Success) { + MACH3LOG_ERROR("Failed to eigen decompose matrix."); + throw MaCh3Exception(__FILE__, __LINE__); + } + + Eigen::VectorXd eigen_val = EigenSolver.eigenvalues(); + Eigen::MatrixXd eigen_vect = EigenSolver.eigenvectors(); + std::vector> evals; + for (int i = 0; i < eigen_val.size(); ++i) { + evals.emplace_back(i, eigen_val[i]); + } + + // sorting is not strictly neccessary but orders the orthogonal parameter + // basis approximately by uncertainty. + std::sort( + evals.begin(), evals.end(), + [](std::pair const &l, std::pair const &r) { + return l.second < r.second; + }); + + double evalsum = 0; + int northo_parameters = 0; + + for (auto &[i, ev] : evals) { + if (std::fabs(ev) > threshold) { + evalsum += std::fabs(ev); + northo_parameters++; + } else { + break; + } + } + + // rows of this matrix correspond to eigenvectors of the input matrix + // we can go from parameters defined in the orthogonal basis back to the + // 'physics' parameters by RowVectOfOrthoParamVals * pca.ortho_to_physics + Eigen::MatrixXd ortho_to_physics = + Eigen::MatrixXd::Zero(northo_parameters, mx_phyiscs_basis.rows()); + + for (int i = 0; i < northo_parameters; ++i) { + for (int j = 0; j < mx_phyiscs_basis.rows(); ++j) { + ortho_to_physics.row(i)[j] = + eigen_vect.col(evals[i].first)[evals[j].first] * + std::sqrt(evals[i].second); + } + } + + return ortho_to_physics; +} diff --git a/Parameters/PCAHandler.cpp b/Parameters/PCAHandler.cpp deleted file mode 100755 index 1e2d69d6e..000000000 --- a/Parameters/PCAHandler.cpp +++ /dev/null @@ -1,729 +0,0 @@ -#include "Parameters/PCAHandler.h" - -// ******************************************** -void PCAHandler::ConstructPCA(Eigen::MatrixXd const &CovMatrix, - const int firstPCAd, const int lastPCAd, - const double eigen_thresh) { - // ******************************************** - FirstPCAdpar = firstPCAd; - LastPCAdpar = lastPCAd; - eigen_threshold = eigen_thresh; - - // Check that covariance matrix exists - if (CovMatrix == NULL) { - MACH3LOG_ERROR("Covariance matrix for has not yet been set"); - MACH3LOG_ERROR("Can not construct PCA until it is set"); - throw MaCh3Exception(__FILE__, __LINE__); - } - - if ((FirstPCAdpar >= CovMatrix.rows()) || (LastPCAdpar >= CovMatrix.rows())) { - MACH3LOG_ERROR("FirstPCAdpar and LastPCAdpar are higher than the number of " - "parameters"); - MACH3LOG_ERROR("first: {} last: {}, params: {}", FirstPCAdpar, LastPCAdpar, - CovMatrix.rows()); - throw MaCh3Exception(__FILE__, __LINE__); - } - if ((FirstPCAdpar < 0) || (LastPCAdpar < 0)) { - MACH3LOG_ERROR( - "FirstPCAdpar and LastPCAdpar are less than 0 but not default -999"); - MACH3LOG_ERROR("first: {} last: {}", FirstPCAdpar, LastPCAdpar); - throw MaCh3Exception(__FILE__, __LINE__); - } - MACH3LOG_INFO("PCAing parameters {} through {} inclusive", FirstPCAdpar, - LastPCAdpar); - int numunpcadpars = CovMatrix->GetNrows() - (LastPCAdpar - FirstPCAdpar + 1); - - M3::EnsureNoOutOfBlockCorrelations(CovMatrix, FirstPCAdpar, LastPCAdpar); - - TMatrixDSym submat( - CovMatrix->GetSub(FirstPCAdpar, LastPCAdpar, FirstPCAdpar, LastPCAdpar)); - - // CW: Calculate how many eigen values this threshold corresponds to - TMatrixDSymEigen eigen(submat); - eigen_values.ResizeTo(eigen.GetEigenValues()); - eigen_vectors.ResizeTo(eigen.GetEigenVectors()); - eigen_values = eigen.GetEigenValues(); - eigen_vectors = eigen.GetEigenVectors(); - double sum = 0; - // Loop over eigen values and sum them up - for (int i = 0; i < eigen_values.GetNrows(); ++i) { - sum += eigen_values(i); - } - nKeptPCApars = eigen_values.GetNrows(); - // CW: Now go through again and see how many eigen values correspond to - // threshold - for (int i = 0; i < eigen_values.GetNrows(); ++i) { - // Get the relative size of the eigen value - double sig = eigen_values(i) / sum; - // Check against the threshold - if (sig < eigen_threshold) { - nKeptPCApars = i; - break; - } - } - NumParPCA = numunpcadpars + nKeptPCApars; - MACH3LOG_INFO("Threshold of {} on eigen values relative sum of eigen value " - "({}) generates {} eigen vectors, plus we have {} unpcad pars, " - "for a total of {}", - eigen_threshold, sum, nKeptPCApars, numunpcadpars, NumParPCA); - - // DB Create array of correct size so eigen_values can be used in - // CorrelateSteps - eigen_values_master = std::vector(NumParPCA, 1.0); - for (int i = FirstPCAdpar; i < FirstPCAdpar + nKeptPCApars; ++i) { - eigen_values_master[i] = eigen_values(i - FirstPCAdpar); - } - - // Now construct the transfer matrices - // These matrices will be as big as number of unPCAd pars plus number of - // eigenvalues kept - TransferMat.ResizeTo(CovMatrix->GetNrows(), NumParPCA); - TransferMatT.ResizeTo(CovMatrix->GetNrows(), NumParPCA); - - // Get a subset of the eigen vector matrix - TMatrixD temp(eigen_vectors.GetSub(0, eigen_vectors.GetNrows() - 1, 0, - nKeptPCApars - 1)); - - // Make transfer matrix which is two blocks of identity with a block of the - // PCA transfer matrix in between - TMatrixD temp2; - temp2.ResizeTo(CovMatrix->GetNrows(), NumParPCA); - - // First set the whole thing to 0 - for (int iRow = 0; iRow < CovMatrix->GetNrows(); iRow++) { - for (int iCol = 0; iCol < NumParPCA; iCol++) { - temp2[iRow][iCol] = 0; - } - } - // Set the first identity block - if (FirstPCAdpar != 0) { - for (int iRow = 0; iRow < FirstPCAdpar; iRow++) { - temp2[iRow][iRow] = 1; - } - } - - // Set the transfer matrix block for the PCAd pars - temp2.SetSub(FirstPCAdpar, FirstPCAdpar, temp); - - // Set the second identity block - if (LastPCAdpar != CovMatrix.rows() - 1) { - for (int iRow = 0; iRow < (CovMatrix.rows() - 1) - LastPCAdpar; iRow++) { - temp2[LastPCAdpar + 1 + iRow][FirstPCAdpar + nKeptPCApars + iRow] = 1; - } - } - - TransferMat = temp2; - // Copy the contents - TransferMatT = TransferMat; - // And then transpose - TransferMatT.T(); - -#ifdef DEBUG_PCA - // KS: Let's dump all useful matrices to properly validate PCA - DebugPCA(sum, temp, submat, CovMatrix->GetNrows()); -#endif - - // Make the PCA parameter arrays - _fParCurrPCA.ResizeTo(NumParPCA); - _fParPropPCA.ResizeTo(NumParPCA); - _fPreFitValuePCA.resize(NumParPCA); - - // KS: make easy map so we could easily find un-decomposed parameters - isDecomposedPCA.resize(NumParPCA); - _fErrorPCA.resize(NumParPCA); - for (int i = 0; i < NumParPCA; ++i) { - _fErrorPCA[i] = 1; - isDecomposedPCA[i] = -1; - } - for (int i = 0; i < FirstPCAdpar; ++i) - isDecomposedPCA[i] = i; - - for (int i = FirstPCAdpar + nKeptPCApars + 1; i < NumParPCA; ++i) - isDecomposedPCA[i] = i + (_fNumPar - NumParPCA); -} - -// ******************************************** -// Update so that current step becomes the previously proposed step -void PCAHandler::AcceptStep() _noexcept_ { - // ******************************************** - // Update the book-keeping for the output -#ifdef MULTITHREAD -#pragma omp parallel for -#endif - for (int i = 0; i < NumParPCA; ++i) { - _fParCurrPCA(i) = _fParPropPCA(i); - } - // Then update the parameter basis - TransferToParam(); -} - -// ************************************************ -// Correlate the steps by setting the proposed step of a parameter to its -// current value + some correlated throw -void PCAHandler::CorrelateSteps( - const std::vector &IndivStepScale, const double GlobalStepScale, - const double *_restrict_ randParams, - const double *_restrict_ corr_throw) _noexcept_ { - // ************************************************ - // Throw around the current step -#ifdef MULTITHREAD -#pragma omp parallel for -#endif - for (int i = 0; i < NumParPCA; ++i) { - if (_fErrorPCA[i] > 0.) { - double IndStepScale = 1.; - // KS: If undecomposed parameter apply individual step scale and Cholesky - // for better acceptance rate - if (isDecomposedPCA[i] >= 0) { - IndStepScale *= IndivStepScale[isDecomposedPCA[i]]; - IndStepScale *= corr_throw[isDecomposedPCA[i]]; - } - // If decomposed apply only random number - else { - IndStepScale *= randParams[i]; - // KS: All PCA-ed parameters have the same step scale - IndStepScale *= IndivStepScale[FirstPCAdpar]; - } - _fParPropPCA(i) = _fParCurrPCA(i) + - GlobalStepScale * IndStepScale * eigen_values_master[i]; - } - } - // Then update the parameter basis - TransferToParam(); -} - -// ******************************************** -// Transfer a parameter variation in the parameter basis to the eigen basis -void PCAHandler::TransferToPCA() { - // ******************************************** - // Make the temporary vectors - TVectorD fParCurr_vec(static_cast(_pCurrVal->size())); - TVectorD fParProp_vec(static_cast(_pCurrVal->size())); - for (int i = 0; i < static_cast(_pCurrVal->size()); ++i) { - fParCurr_vec(i) = (*_pCurrVal)[i]; - fParProp_vec(i) = (*_pPropVal)[i]; - } - - _fParCurrPCA = TransferMatT * fParCurr_vec; - _fParPropPCA = TransferMatT * fParProp_vec; -} - -// ******************************************** -void PCAHandler::SetInitialParameters(std::vector &IndStepScale) { - // ******************************************** - TransferToPCA(); - for (int i = 0; i < NumParPCA; ++i) { - _fPreFitValuePCA[i] = _fParCurrPCA(i); - } - // DB Set Individual Step scale for PCA parameters to the LastPCAdpar - // fIndivStepScale because the step scale for those parameters is set by - // 'eigen_values[i]' but needs an overall step scale - // However, individual step scale for non-PCA parameters needs to be set - // correctly - for (int i = FirstPCAdpar; i <= LastPCAdpar; i++) { - IndStepScale[i] = IndStepScale[LastPCAdpar - 1]; - } -} - -// ******************************************** -// Transfer a parameter variation in the eigen basis to the parameter basis -void PCAHandler::TransferToParam() { - // ******************************************** - // Make the temporary vectors - TVectorD fParProp_vec = TransferMat * _fParPropPCA; - TVectorD fParCurr_vec = TransferMat * _fParCurrPCA; -#ifdef MULTITHREAD -#pragma omp parallel for -#endif - for (int i = 0; i < static_cast(_pCurrVal->size()); ++i) { - (*_pPropVal)[i] = fParProp_vec(i); - (*_pCurrVal)[i] = fParCurr_vec(i); - } -} - -// ******************************************** -// Throw the proposed parameter by mag sigma. -void PCAHandler::ThrowParProp(const double mag, - const double *_restrict_ randParams) { - // ******************************************** - for (int i = 0; i < NumParPCA; i++) { - if (_fErrorPCA[i] > 0.) { - _fParPropPCA(i) = _fParCurrPCA(i) + mag * randParams[i]; - } - } - TransferToParam(); -} - -// ******************************************** -// Throw the proposed parameter by mag sigma. -void PCAHandler::ThrowParCurr(const double mag, - const double *_restrict_ randParams) { - // ******************************************** - for (int i = 0; i < NumParPCA; i++) { - if (_fErrorPCA[i] > 0.) { - _fParPropPCA(i) = mag * randParams[i]; - } - } - TransferToParam(); -} - -// ******************************************** -void PCAHandler::Print() { - // ******************************************** - MACH3LOG_INFO("PCA:"); - for (int i = 0; i < NumParPCA; ++i) { - MACH3LOG_INFO("PCA {:<2} Current: {:<10.2f} Proposed: {:<10.2f}", i, - _fParCurrPCA(i), _fParPropPCA(i)); - } -} - -// ******************************************** -void PCAHandler::SetBranches(TTree &tree, bool SaveProposal, - const std::vector &Names) { - // ******************************************** - for (int i = 0; i < NumParPCA; ++i) { - tree.Branch(Form("%s_PCA", Names[i].c_str()), - &_fParCurrPCA.GetMatrixArray()[i], - Form("%s_PCA/D", Names[i].c_str())); - } - - if (SaveProposal) { - for (int i = 0; i < NumParPCA; ++i) { - tree.Branch(Form("%s_PCA_Prop", Names[i].c_str()), - &_fParPropPCA.GetMatrixArray()[i], - Form("%s_PCA_Prop/D", Names[i].c_str())); - } - } -} - -// ******************************************** -void PCAHandler::ToggleFixAllParameters(const std::vector &Names) { - // ******************************************** - for (int i = 0; i < NumParPCA; i++) - ToggleFixParameter(i, Names); -} - -// ******************************************** -void PCAHandler::ToggleFixParameter(const int i, - const std::vector &Names) { - // ******************************************** - int isDecom = -1; - for (int im = 0; im < NumParPCA; ++im) { - if (isDecomposedPCA[im] == i) { - isDecom = im; - } - } - if (isDecom < 0) { - MACH3LOG_ERROR("Parameter {} is PCA decomposed can't fix this", Names[i]); - // throw MaCh3Exception(__FILE__ , __LINE__ ); - } else { - _fErrorPCA[isDecom] *= -1.0; - if (IsParameterFixedPCA(i)) - MACH3LOG_INFO("Setting un-decomposed {}(parameter {}/{} in PCA base) to " - "fixed at {}", - Names[i], i, isDecom, (*_pCurrVal)[i]); - else - MACH3LOG_INFO( - "Setting un-decomposed {}(parameter {}/{} in PCA base) free", - Names[i], i, isDecom); - } -} - -// ******************************************** -void PCAHandler::ThrowParameters( - const std::vector> &random_number, - double **throwMatrixCholDecomp, double *randParams, double *corr_throw, - const std::vector &fPreFitValue, - const std::vector &fLowBound, const std::vector &fUpBound, - const int _fNumPar) { - // ******************************************** - // KS: Do not multithread! - for (int i = 0; i < NumParPCA; ++i) { - // Check if parameter is fixed first: if so don't randomly throw - if (IsParameterFixedPCA(i)) - continue; - - if (!IsParameterDecomposed(i)) { - (*_pPropVal)[i] = fPreFitValue[i] + corr_throw[i]; - int throws = 0; - // Try again if we the initial parameter proposal falls outside of the - // range of the parameter - while ((*_pPropVal)[i] > fUpBound[i] || (*_pPropVal)[i] < fLowBound[i]) { - randParams[i] = random_number[M3::GetThreadIndex()]->Gaus(0, 1); - const double corr_throw_single = M3::MatrixVectorMultiSingle( - throwMatrixCholDecomp, randParams, _fNumPar, i); - (*_pPropVal)[i] = fPreFitValue[i] + corr_throw_single; - if (throws > 10000) { - // KS: Since we are multithreading there is danger that those messages - // will be all over the place, small price to pay for faster code - MACH3LOG_WARN("Tried {} times to throw parameter {} but failed", - throws, i); - MACH3LOG_WARN("Setting _fPropVal: {} to {}", (*_pPropVal)[i], - fPreFitValue[i]); - MACH3LOG_WARN("I live at {}:{}", __FILE__, __LINE__); - (*_pPropVal)[i] = fPreFitValue[i]; - } - throws++; - } - (*_pCurrVal)[i] = (*_pPropVal)[i]; - - } else { - // KS: We have to multiply by number of parameters in PCA base - SetParPropPCA(i, GetPreFitValuePCA(i) + randParams[i] * - eigen_values_master[i] * - (LastPCAdpar - FirstPCAdpar)); - SetParCurrPCA(i, GetParPropPCA(i)); - } - } // end of parameter loop - - /// @todo KS: We don't check if param is out of bounds. This is more - /// problematic for PCA params. - for (int i = 0; i < _fNumPar; ++i) { - (*_pPropVal)[i] = - std::max(fLowBound[i], std::min((*_pPropVal)[i], fUpBound[i])); - (*_pCurrVal)[i] = (*_pPropVal)[i]; - } -} - -#ifdef DEBUG_PCA -#pragma GCC diagnostic ignored "-Wfloat-conversion" -// ******************************************** -// KS: Let's dump all useful matrices to properly validate PCA -void PCAHandler::DebugPCA(const double sum, TMatrixD temp, TMatrixDSym submat, - int NumPar) { - // ******************************************** - (void)submat; // This is used if DEBUG_PCA==2, this hack is to avoid compiler - // warnings - TFile *PCA_Debug = new TFile("Debug_PCA.root", "RECREATE"); - PCA_Debug->cd(); - - bool PlotText = true; - // KS: If we have more than 200 plot becomes unreadable :( - if (NumPar > 200) - PlotText = false; - - TH1D *heigen_values = - new TH1D("eigen_values", "Eigen Values", eigen_values.GetNrows(), 0.0, - eigen_values.GetNrows()); - TH1D *heigen_cumulative = - new TH1D("heigen_cumulative", "heigen_cumulative", - eigen_values.GetNrows(), 0.0, eigen_values.GetNrows()); - TH1D *heigen_frac = - new TH1D("heigen_fractional", "heigen_fractional", - eigen_values.GetNrows(), 0.0, eigen_values.GetNrows()); - heigen_values->GetXaxis()->SetTitle("Eigen Vector"); - heigen_values->GetYaxis()->SetTitle("Eigen Value"); - - double Cumulative = 0; - for (int i = 0; i < eigen_values.GetNrows(); i++) { - heigen_values->SetBinContent(i + 1, (eigen_values)(i)); - heigen_cumulative->SetBinContent(i + 1, - (eigen_values)(i) / sum + Cumulative); - heigen_frac->SetBinContent(i + 1, (eigen_values)(i) / sum); - Cumulative += (eigen_values)(i) / sum; - } - heigen_values->Write("heigen_values"); - eigen_values.Write("eigen_values"); - heigen_cumulative->Write("heigen_values_cumulative"); - heigen_frac->Write("heigen_values_frac"); - - TH2D *heigen_vectors = new TH2D(eigen_vectors); - heigen_vectors->GetXaxis()->SetTitle("Parameter in Normal Base"); - heigen_vectors->GetYaxis()->SetTitle("Parameter in Decomposed Base"); - heigen_vectors->Write("heigen_vectors"); - eigen_vectors.Write("eigen_vectors"); - - TH2D *SubsetPCA = new TH2D(temp); - SubsetPCA->GetXaxis()->SetTitle("Parameter in Normal Base"); - SubsetPCA->GetYaxis()->SetTitle("Parameter in Decomposed Base"); - - SubsetPCA->Write("hSubsetPCA"); - temp.Write("SubsetPCA"); - TH2D *hTransferMat = new TH2D(TransferMat); - hTransferMat->GetXaxis()->SetTitle("Parameter in Normal Base"); - hTransferMat->GetYaxis()->SetTitle("Parameter in Decomposed Base"); - TH2D *hTransferMatT = new TH2D(TransferMatT); - - hTransferMatT->GetXaxis()->SetTitle("Parameter in Decomposed Base"); - hTransferMatT->GetYaxis()->SetTitle("Parameter in Normal Base"); - - hTransferMat->Write("hTransferMat"); - TransferMat.Write("TransferMat"); - hTransferMatT->Write("hTransferMatT"); - TransferMatT.Write("TransferMatT"); - - TCanvas *c1 = new TCanvas("c1", " ", 0, 0, 1024, 1024); - c1->SetBottomMargin(0.1); - c1->SetTopMargin(0.05); - c1->SetRightMargin(0.05); - c1->SetLeftMargin(0.12); - c1->SetGrid(); - - gStyle->SetPaintTextFormat("4.1f"); - gStyle->SetOptFit(0); - gStyle->SetOptStat(0); - // Make pretty correlation colors (red to blue) - const int NRGBs = 5; - TColor::InitializeColors(); - Double_t stops[NRGBs] = {0.00, 0.25, 0.50, 0.75, 1.00}; - Double_t red[NRGBs] = {0.00, 0.25, 1.00, 1.00, 0.50}; - Double_t green[NRGBs] = {0.00, 0.25, 1.00, 0.25, 0.00}; - Double_t blue[NRGBs] = {0.50, 1.00, 1.00, 0.25, 0.00}; - TColor::CreateGradientColorTable(5, stops, red, green, blue, 255); - gStyle->SetNumberContours(255); - - double maxz = 0; - double minz = 0; - - c1->Print("Debug_PCA.pdf["); - TLine *EigenLine = - new TLine(nKeptPCApars, 0, nKeptPCApars, heigen_cumulative->GetMaximum()); - EigenLine->SetLineColor(kPink); - EigenLine->SetLineWidth(2); - EigenLine->SetLineStyle(kSolid); - - TText *text = new TText(0.5, 0.5, Form("Threshold = %g", eigen_threshold)); - text->SetTextFont(43); - text->SetTextSize(40); - - heigen_values->SetLineColor(kRed); - heigen_values->SetLineWidth(2); - heigen_cumulative->SetLineColor(kGreen); - heigen_cumulative->SetLineWidth(2); - heigen_frac->SetLineColor(kBlue); - heigen_frac->SetLineWidth(2); - - c1->SetLogy(); - heigen_values->SetMaximum(heigen_cumulative->GetMaximum() + - heigen_cumulative->GetMaximum() * 0.4); - heigen_values->Draw(); - heigen_frac->Draw("SAME"); - heigen_cumulative->Draw("SAME"); - EigenLine->Draw("Same"); - text->DrawTextNDC(0.42, 0.84, Form("Threshold = %g", eigen_threshold)); - - TLegend *leg = new TLegend(0.2, 0.2, 0.6, 0.5); - leg->SetTextSize(0.04); - leg->AddEntry(heigen_values, "Absolute", "l"); - leg->AddEntry(heigen_frac, "Fractional", "l"); - leg->AddEntry(heigen_cumulative, "Cumulative", "l"); - - leg->SetLineColor(0); - leg->SetLineStyle(0); - leg->SetFillColor(0); - leg->SetFillStyle(0); - leg->Draw("Same"); - - c1->Print("Debug_PCA.pdf"); - c1->SetRightMargin(0.15); - c1->SetLogy(0); - delete EigenLine; - delete leg; - delete text; - delete heigen_values; - delete heigen_frac; - delete heigen_cumulative; - - heigen_vectors->SetMarkerSize(0.2); - minz = heigen_vectors->GetMinimum(); - if (fabs(0 - maxz) > fabs(0 - minz)) - heigen_vectors->GetZaxis()->SetRangeUser(0 - fabs(0 - maxz), - 0 + fabs(0 - maxz)); - else - heigen_vectors->GetZaxis()->SetRangeUser(0 - fabs(0 - minz), - 0 + fabs(0 - minz)); - if (PlotText) - heigen_vectors->Draw("COLZ TEXT"); - else - heigen_vectors->Draw("COLZ"); - - TLine *Eigen_Line = - new TLine(0, nKeptPCApars, LastPCAdpar - FirstPCAdpar, nKeptPCApars); - Eigen_Line->SetLineColor(kGreen); - Eigen_Line->SetLineWidth(2); - Eigen_Line->SetLineStyle(kDotted); - Eigen_Line->Draw("SAME"); - c1->Print("Debug_PCA.pdf"); - delete Eigen_Line; - - SubsetPCA->SetMarkerSize(0.2); - minz = SubsetPCA->GetMinimum(); - if (fabs(0 - maxz) > fabs(0 - minz)) - SubsetPCA->GetZaxis()->SetRangeUser(0 - fabs(0 - maxz), 0 + fabs(0 - maxz)); - else - SubsetPCA->GetZaxis()->SetRangeUser(0 - fabs(0 - minz), 0 + fabs(0 - minz)); - if (PlotText) - SubsetPCA->Draw("COLZ TEXT"); - else - SubsetPCA->Draw("COLZ"); - c1->Print("Debug_PCA.pdf"); - delete SubsetPCA; - - hTransferMat->SetMarkerSize(0.15); - minz = hTransferMat->GetMinimum(); - if (fabs(0 - maxz) > fabs(0 - minz)) - hTransferMat->GetZaxis()->SetRangeUser(0 - fabs(0 - maxz), - 0 + fabs(0 - maxz)); - else - hTransferMat->GetZaxis()->SetRangeUser(0 - fabs(0 - minz), - 0 + fabs(0 - minz)); - if (PlotText) - hTransferMat->Draw("COLZ TEXT"); - else - hTransferMat->Draw("COLZ"); - c1->Print("Debug_PCA.pdf"); - delete hTransferMat; - - hTransferMatT->SetMarkerSize(0.15); - minz = hTransferMatT->GetMinimum(); - if (fabs(0 - maxz) > fabs(0 - minz)) - hTransferMatT->GetZaxis()->SetRangeUser(0 - fabs(0 - maxz), - 0 + fabs(0 - maxz)); - else - hTransferMatT->GetZaxis()->SetRangeUser(0 - fabs(0 - minz), - 0 + fabs(0 - minz)); - if (PlotText) - hTransferMatT->Draw("COLZ TEXT"); - else - hTransferMatT->Draw("COLZ"); - c1->Print("Debug_PCA.pdf"); - delete hTransferMatT; - -// KS: Crosscheck against Eigen library -#if DEBUG_PCA == 2 - Eigen::MatrixXd Submat_Eigen(submat.GetNrows(), submat.GetNcols()); - -#ifdef MULTITHREAD -#pragma omp parallel for -#endif - for (int i = 0; i < submat.GetNrows(); i++) { - for (int j = 0; j < submat.GetNcols(); j++) { - Submat_Eigen(i, j) = (submat)(i, j); - } - } - Eigen::EigenSolver EigenSolver; - EigenSolver.compute(Submat_Eigen); - Eigen::VectorXd eigen_val = EigenSolver.eigenvalues().real(); - Eigen::MatrixXd eigen_vect = EigenSolver.eigenvectors().real(); - std::vector> eigen_vectors_and_values; - double Sum_Eigen = 0; - for (int i = 0; i < eigen_val.size(); i++) { - std::tuple vec_and_val(eigen_val[i], - eigen_vect.row(i)); - eigen_vectors_and_values.push_back(vec_and_val); - Sum_Eigen += eigen_val[i]; - } - std::sort(eigen_vectors_and_values.begin(), eigen_vectors_and_values.end(), - [&](const std::tuple &a, - const std::tuple &b) -> bool { - return std::get<0>(a) > std::get<0>(b); - }); - int index = 0; - for (auto const vect : eigen_vectors_and_values) { - eigen_val(index) = std::get<0>(vect); - eigen_vect.row(index) = std::get<1>(vect); - index++; - } - TH1D *heigen_values_Eigen = new TH1D("eig_values", "Eigen Values", - eigen_val.size(), 0.0, eigen_val.size()); - TH1D *heigen_cumulative_Eigen = - new TH1D("eig_cumulative", "heigen_cumulative", eigen_val.size(), 0.0, - eigen_val.size()); - TH1D *heigen_frac_Eigen = new TH1D("eig_fractional", "heigen_fractional", - eigen_val.size(), 0.0, eigen_val.size()); - heigen_values_Eigen->GetXaxis()->SetTitle("Eigen Vector"); - heigen_values_Eigen->GetYaxis()->SetTitle("Eigen Value"); - - double Cumulative_Eigen = 0; - for (int i = 0; i < eigen_val.size(); i++) { - heigen_values_Eigen->SetBinContent(i + 1, eigen_val(i)); - heigen_cumulative_Eigen->SetBinContent(i + 1, eigen_val(i) / sum + - Cumulative_Eigen); - heigen_frac_Eigen->SetBinContent(i + 1, eigen_val(i) / sum); - Cumulative_Eigen += eigen_val(i) / sum; - } - heigen_values_Eigen->SetLineColor(kRed); - heigen_values_Eigen->SetLineWidth(2); - heigen_cumulative_Eigen->SetLineColor(kGreen); - heigen_cumulative_Eigen->SetLineWidth(2); - heigen_frac_Eigen->SetLineColor(kBlue); - heigen_frac_Eigen->SetLineWidth(2); - - c1->SetLogy(); - heigen_values_Eigen->SetMaximum(heigen_cumulative_Eigen->GetMaximum() + - heigen_cumulative_Eigen->GetMaximum() * 0.4); - heigen_values_Eigen->Draw(); - heigen_cumulative_Eigen->Draw("SAME"); - heigen_frac_Eigen->Draw("SAME"); - - TLegend *leg_Eigen = new TLegend(0.2, 0.2, 0.6, 0.5); - leg_Eigen->SetTextSize(0.04); - leg_Eigen->AddEntry(heigen_values_Eigen, "Absolute", "l"); - leg_Eigen->AddEntry(heigen_frac_Eigen, "Fractional", "l"); - leg_Eigen->AddEntry(heigen_cumulative_Eigen, "Cumulative", "l"); - - leg_Eigen->SetLineColor(0); - leg_Eigen->SetLineStyle(0); - leg_Eigen->SetFillColor(0); - leg_Eigen->SetFillStyle(0); - leg_Eigen->Draw("Same"); - - c1->Print("Debug_PCA.pdf"); - c1->SetLogy(0); - delete heigen_values_Eigen; - delete heigen_cumulative_Eigen; - delete heigen_frac_Eigen; - delete leg_Eigen; - - TH2D *heigen_vectors_Eigen = - new TH2D("Eigen_Vectors", "Eigen_Vectors", eigen_val.size(), 0.0, - eigen_val.size(), eigen_val.size(), 0.0, eigen_val.size()); - - for (int i = 0; i < eigen_val.size(); i++) { - for (int j = 0; j < eigen_val.size(); j++) { - // KS: +1 because there is offset in histogram relative to TMatrix - heigen_vectors_Eigen->SetBinContent(i + 1, j + 1, eigen_vect(i, j)); - } - } - heigen_vectors_Eigen->GetXaxis()->SetTitle("Parameter in Normal Base"); - heigen_vectors_Eigen->GetYaxis()->SetTitle("Parameter in Decomposed Base"); - heigen_vectors_Eigen->SetMarkerSize(0.15); - minz = heigen_vectors_Eigen->GetMinimum(); - if (fabs(0 - maxz) > fabs(0 - minz)) - heigen_vectors_Eigen->GetZaxis()->SetRangeUser(0 - fabs(0 - maxz), - 0 + fabs(0 - maxz)); - else - heigen_vectors_Eigen->GetZaxis()->SetRangeUser(0 - fabs(0 - minz), - 0 + fabs(0 - minz)); - - if (PlotText) - heigen_vectors_Eigen->Draw("COLZ TEXT"); - else - heigen_vectors_Eigen->Draw("COLZ"); - c1->Print("Debug_PCA.pdf"); - - heigen_vectors->SetTitle("ROOT minus Eigen"); - heigen_vectors->Add(heigen_vectors_Eigen, -1.); - minz = heigen_vectors->GetMinimum(); - if (fabs(0 - maxz) > fabs(0 - minz)) - heigen_vectors->GetZaxis()->SetRangeUser(0 - fabs(0 - maxz), - 0 + fabs(0 - maxz)); - else - heigen_vectors->GetZaxis()->SetRangeUser(0 - fabs(0 - minz), - 0 + fabs(0 - minz)); - if (PlotText) - heigen_vectors->Draw("COLZ TEXT"); - else - heigen_vectors->Draw("COLZ"); - c1->Print("Debug_PCA.pdf"); - delete heigen_vectors_Eigen; - -#endif // end if Eigen enabled - delete heigen_vectors; - - c1->Print("Debug_PCA.pdf]"); - delete c1; - PCA_Debug->Close(); - delete PCA_Debug; -} -#endif diff --git a/Parameters/PCAHandler.h b/Parameters/PCAHandler.h deleted file mode 100644 index 89d689788..000000000 --- a/Parameters/PCAHandler.h +++ /dev/null @@ -1,262 +0,0 @@ -#pragma once - -// MaCh3 Includes -#include "Manager/Manager.h" -#include "Parameters/ParameterHandlerUtils.h" - -#ifdef DEBUG - #define DEBUG_PCA 1 -#endif - -#ifdef DEBUG_PCA -//KS: When debugging we produce some fancy plots, but we don't need it during normal work flow -#include "TCanvas.h" -#include "TROOT.h" -#include "TStyle.h" -#include "TColor.h" -#include "TLine.h" -#include "TText.h" -#include "TLegend.h" - -#if DEBUG_PCA == 2 -#include "Eigen/Eigenvalues" -#endif - -#endif - -/// @brief Class responsible for handling Principal Component Analysis (PCA) of covariance matrix -/// @author Clarence Wret -/// -/// @section introPCA Introduction -/// -/// This class applies Principal Component Analysis (PCA) to covariance matrices -/// using eigen-decomposition. The main benefit is that PCA rotates the parameter -/// space into a basis where parameters are uncorrelated. In practice, this helps -/// MCMC fits move more efficiently, since correlated directions in the original -/// space often slow down convergence. -/// -/// @section dimredPCA Dimensionality Reduction -/// -/// PCA makes it possible to drop directions in parameter space with very small -/// eigenvalues. These correspond to combinations of parameters that are poorly -/// constrained or dominated by statistical noise. By applying a threshold -/// (e.g. discarding eigenvalues smaller than `1e-4` of the largest one), the -/// effective number of parameters can be reduced. -/// -/// This is particularly useful in large MCMC fits, where many parameters may -/// contribute little information. Removing them reduces dimensionality while -/// keeping the dominant structure of the parameter space intact, often leading -/// to faster and more stable fits. -/// -/// @section partialPCA Partial Decomposition -/// -/// PCA does not need to be applied to the full covariance matrix. This class -/// supports eigen-decomposition of only a submatrix. This is useful when only a -/// subset of parameters is strongly correlated and benefits from PCA, while the -/// rest remain in the original parameter basis. -/// -/// -/// @see For more details, visit the [Wiki](https://github.com/mach3-software/MaCh3/wiki/03.-Eigen-Decomposition-%E2%80%90-PCA). -class PCAHandler{ - public: - /// @brief Constructor - PCAHandler(){}; - - /// @brief Destructor - virtual ~PCAHandler(){}; - - /// @brief KS: Print info about PCA parameters - void Print() const; - /// @brief Retrieve number of parameters in PCA base - int GetNumberPCAedParameters() const { return NumParPCA; } - - /// @brief CW: Calculate eigen values, prepare transition matrices and remove param based on defined threshold - /// @param CovMatrix Symmetric covariance matrix used for eigen decomposition. - /// @param firstPCAd Index of the first PCA component to include. - /// @param lastPCAd Index of the last PCA component to include. - /// @param eigen_thresh Threshold for eigenvalues below which parameters are discarded. - /// @param _fNumPar Total number of parameters in the original (non-PCA) basis. - void ConstructPCA(Eigen::MatrixXd const & CovMatrix, const int firstPCAd, const int lastPCAd, - const double eigen_thresh); - - /// @brief Transfer param values from normal base to PCA base - void SetUnrotatedParameterValues(Eigen::VectorXd const &vals); - /// @brief Transfer param values from PCA base to normal base - void TransferToParam(); - - /// @brief Throw the parameters according to the covariance matrix. This shouldn't be used in MCMC code ase it can break Detailed Balance; - void ThrowParameters(const std::vector>& random_number, - double** throwMatrixCholDecomp, - double* randParams, - double* corr_throw, - const std::vector& fPreFitValue, - const std::vector& fLowBound, - const std::vector& fUpBound, - int _fNumPar); - - /// @brief Accept this step - void AcceptStep() _noexcept_; - /// @brief Use Cholesky throw matrix for better step proposal - void CorrelateSteps(const std::vector& IndivStepScale, - const double GlobalStepScale, - const double* _restrict_ randParams, - const double* _restrict_ corr_throw) _noexcept_; - - /// @brief KS: Transfer the starting parameters to the PCA basis, you don't want to start with zero.. - /// @param IndStepScale Per parameter step scale, we set so each PCA param uses same step scale [eigen value takes care of size] - void SetInitialParameters(std::vector& IndStepScale); - /// @brief Throw the proposed parameter by mag sigma. - void ThrowParProp(const double mag, const double* _restrict_ randParams); - /// @brief Helper function to throw the current parameter by mag sigma. - void ThrowParCurr(const double mag, const double* _restrict_ randParams); - /// @brief set branches for output file - /// @ingroup ParameterHandlerSetters - void SetBranches(TTree &tree, bool SaveProposal, const std::vector& Names); - /// @brief fix parameters at prior values - /// @ingroup ParameterHandlerSetters - void ToggleFixAllParameters(const std::vector& Names); - /// @brief fix parameters at prior values - /// @ingroup ParameterHandlerSetters - void ToggleFixParameter(const int i, const std::vector& Names); - - /// @brief Set values for PCA parameters in PCA base - /// @param pars vector with new values of PCA params - /// @ingroup ParameterHandlerSetters - void SetParametersPCA(const std::vector &pars) { - if (int(pars.size()) != NumParPCA) { - MACH3LOG_ERROR("Parameter arrays of incompatible size! Not changing parameters! has size {} but was expecting {}", pars.size(), NumParPCA); - throw MaCh3Exception(__FILE__ , __LINE__ ); - } - int parsSize = int(pars.size()); - for (int i = 0; i < parsSize; i++) { - _fParPropPCA(i) = pars[i]; - } - //KS: Transfer to normal base - TransferToParam(); - } - - /// @brief Is parameter fixed in PCA base or not - /// @param i Parameter index - /// @ingroup ParameterHandlerGetters - bool IsParameterFixedPCA(const int i) const { - if (_fErrorPCA[i] < 0) { return true; } - else { return false; } - } - - /// @brief Get eigen vectors of covariance matrix, only works with PCA - /// @ingroup ParameterHandlerGetters - const TMatrixD GetEigenVectors() const { - return eigen_vectors; - } - - /// @brief Set proposed value for parameter in PCA base - /// @param i Parameter index - /// @param value new value - /// @ingroup ParameterHandlerSetters - void SetParPropPCA(const int i, const double value) { - _fParPropPCA(i) = value; - // And then transfer back to the parameter basis - TransferToParam(); - } - /// @brief Set current value for parameter in PCA base - /// @param i Parameter index - /// @param value new value - /// @ingroup ParameterHandlerSetters - void SetParCurrPCA(const int i, const double value) { - _fParCurrPCA(i) = value; - // And then transfer back to the parameter basis - TransferToParam(); - } - - /// @brief Get current parameter value using PCA - /// @param i Parameter index - /// @ingroup ParameterHandlerGetters - double GetParPropPCA(const int i) const { - return _fParPropPCA(i); - } - - /// @brief Get current parameter value using PCA - /// @param i Parameter index - /// @ingroup ParameterHandlerGetters - double GetPreFitValuePCA(const int i) const { - return _fPreFitValuePCA[i]; - } - - /// @brief Get current parameter value using PCA - /// @param i Parameter index - /// @ingroup ParameterHandlerGetters - double GetParCurrPCA(const int i) const { - return _fParCurrPCA(i); - } - - /// @brief Get transfer matrix allowing to go from PCA base to normal base - /// @ingroup ParameterHandlerGetters - const TMatrixD GetTransferMatrix() const { - return TransferMat; - } - - /// @brief Get eigen values for all parameters, if you want for decomposed only parameters use GetEigenValuesMaster - /// @ingroup ParameterHandlerGetters - const TVectorD GetEigenValues() const { - return eigen_values; - } - /// @brief Get eigen value of only decomposed parameters, if you want for all parameters use GetEigenValues - /// @ingroup ParameterHandlerGetters - const std::vector GetEigenValuesMaster() const { - return eigen_values_master; - } - - /// @brief Check if parameter in PCA base is decomposed or not - /// @param i Parameter index - /// @ingroup ParameterHandlerGetters - bool IsParameterDecomposed(const int i) const { - if(isDecomposedPCA[i] >= 0) return false; - else return true; - } - - #ifdef DEBUG_PCA - /// @brief KS: Let's dump all useful matrices to properly validate PCA - void DebugPCA(const double sum, TMatrixD temp, TMatrixDSym submat, int NumPar); - #endif - - private: - - /// Prefit value for PCA params - std::vector _fPreFitValuePCA; - /// CW: Current parameter value in PCA base - TVectorD _fParPropPCA; - /// CW: Proposed parameter value in PCA base - TVectorD _fParCurrPCA; - /// Tells if parameter is fixed in PCA base or not - std::vector _fErrorPCA; - /// If param is decomposed this will return -1, if not this will return enumerator to param in normal base. This way we can map stuff like step scale etc between normal base and undecomposed param in eigen base. - std::vector isDecomposedPCA; - /// Number of parameters in PCA base - int NumParPCA; - - /// Matrix used to converting from PCA base to normal base - TMatrixD TransferMat; - /// Matrix used to converting from normal base to PCA base - TMatrixD TransferMatT; - /// Eigen value only of particles which are being decomposed - TVectorD eigen_values; - /// Eigen vectors only of params which are being decomposed - TMatrixD eigen_vectors; - /// Eigen values which have dimension equal to _fNumParPCA, and can be used in CorrelateSteps - std::vector eigen_values_master; - - /// Total number that remained after applying PCA Threshold - int nKeptPCApars; - /// Index of the first param that is being decomposed - int FirstPCAdpar; - /// Index of the last param that is being decomposed - int LastPCAdpar; - /// CW: Threshold based on which we remove parameters in eigen base - double eigen_threshold; - - /// Pointer to current value of the parameter - std::vector* _pCurrVal; - /// Pointer to proposed value of the parameter - std::vector* _pPropVal; -}; - diff --git a/Parameters/ParameterHandlerBase.cpp b/Parameters/ParameterHandlerBase.cpp index ae9745443..5f6fd1449 100644 --- a/Parameters/ParameterHandlerBase.cpp +++ b/Parameters/ParameterHandlerBase.cpp @@ -1,28 +1,25 @@ #include "Parameters/ParameterHandlerBase.h" #include -ParameterHandlerBase::ParameterHandlerBase() - : pca.enabled{false}, -special_proposal.enabled{false}, rng.gaus(0), settings.use_adaptive{false}, -settings.PrintLength{35} {} +ParameterHandlerBase::ParameterHandlerBase() { + pca.enabled = false; + special_proposal.enabled = false; + rng.gaus = std::normal_distribution(0); + rng.unif = std::uniform_real_distribution(0, 1); + settings.use_adaptive = false; + settings.PrintLength = 35; +} // ******************************************** ParameterHandlerBase::ParameterHandlerBase(std::string const &name, - std::string const &inputFile) + std::string const &file) : ParameterHandlerBase() { // ******************************************** - MACH3LOG_DEBUG("Constructing instance of ParameterHandler, named: {}", name); SetName(name); - config.inputFile = file; -} - -// ******************************************** -ParameterHandlerBase::ParameterHandlerBase(std::string name, std::string file, - double threshold, int FirstPCA, - int LastPCA) - : ParameterHandlerBase(name, file) { - // ******************************************** + config.inputFiles = { + file, + }; // Set the covariance matrix from input ROOT file (e.g. flux, ND280, NIWG) TFile infile(file.c_str(), "READ"); @@ -41,112 +38,30 @@ ParameterHandlerBase::ParameterHandlerBase(std::string name, std::string file, throw MaCh3Exception(__FILE__, __LINE__); } - params.covariance = M3::MakePosDef(M3::ROOTToEigen(*CovMat)); - - for (int iThread = 0; iThread < M3::GetNThreads(); iThread++) { - random_number.emplace_back(0); - } + params.covariance = M3::MakeMatrixPosDef(M3::ROOTToEigen(*CovMat)); MACH3LOG_INFO("Created covariance matrix named: {}", GetName()); MACH3LOG_INFO("from file: {}", file); } -void ParameterHandlerBase::AddParameters(std::vector const ¶ms) { - - for (auto const &p : params) { - params.name.push_back(p.name); - params.fancy_name.push_back(p.fancy_name); - params.samples.push_back(p.affected_samples); - } - - size_t new_block_start = params.prefit.size(); - - params.prefit.conservativeResize(params.name.size()); - params.error.conservativeResize(params.name.size()); - params.lowbound.conservativeResize(params.name.size()); - params.upbound.conservativeResize(params.name.size()); - params.flatprior.conservativeResize(params.name.size()); - params.fixed.conservativeResize(params.name.size()); - steps.scale.conservativeResize(params.name.size()); - - params.covariance.conservativeResize(params.name.size(), params.name.size()); - - for (size_t i = 0; i < params.size(); ++i) { - params.prefit[new_block_start + i] = params[i].prefit; - params.error[new_block_start + i] = params[i].error; - params.lowbound[new_block_start + i] = params[i].bounds[0]; - params.upbound[new_block_start + i] = params[i].bounds[1]; - params.flatprior[new_block_start + i] = params[i].flatprior; - params.fixed[new_block_start + i] = false; - steps.scale[new_block_start + i] = params[i].stepscale; - params.covariance(new_block_start + i, new_block_start + i) = - params[i].error * params[i].error; - } - - params.inv_covariance = MatrixXd(0); -} - -void ParameterHandlerBase::SetParameterCorrelation(int pidi, int pidj, - double corr) { - if (pidi == pidj) { - MACH3LOG_ERROR( - "AddParameterCorrelation cannot be used to set covariance " - "matrix diagonal elements: ({0},{0}) attempted to be set to {1}", - pidi, corr); - throw MaCh3Exception(__FILE__, __LINE__); - } - - params.covariance(pidi, pidj) = - corr * - std::sqrt(params.covariance(pidi, pidi) * params.covariance(pidj, pidj)); - params.covariance(pidj, pidi) = params.covariance(pidi, pidj); - - params.inv_covariance = MatrixXd(0); -} - -void ParameterHandlerBase::SetParameterAllCorrelations( - int paramid, std::map const &correlations) { - - params.covariance.row(paramid) = - Eigen::VectorXd::Zeros(params.covariance.rows()); - params.covariance.col(paramid) = - Eigen::VectorXd::Zeros(params.covariance.rows()); - - params.covariance(paramid, paramid) = - params.error[paramid] * params.error[paramid]; - - for (auto const &[other_name, corr] : correlations) { - SetParameterCorrelation(paramid, GetParIndex(other_name), corr); - } -} - // ******************************************** ParameterHandlerBase::ParameterHandlerBase( - const std::vector &YAMLFiles, std::string name, - double threshold, int FirstPCA, int LastPCA) - : ParameterHandlerBase(name, YAMLFiles[0]) { + const std::vector &YAMLFiles, std::string const &name) + : ParameterHandlerBase() { // ******************************************** + SetName(name); + config.inputFiles = YAMLFiles; + MACH3LOG_INFO("Constructing instance of ParameterHandler using: "); - for (auto const &yf : YAMLFile) { + for (auto const &yf : config.inputFiles) { MACH3LOG_INFO(" {}", yf); } MACH3LOG_INFO("as an input."); - settings.pca = true; - if (threshold < 0 || threshold >= 1) { - MACH3LOG_INFO("Principal component analysis but given the threshold for " - "the principal components to be less than 0, or greater than " - "(or equal to) 1. This will not work"); - MACH3LOG_INFO("Please specify a number between 0 and 1"); - MACH3LOG_INFO("You specified: {}", threshold); - MACH3LOG_INFO("Disabling PCA..."); - settings.pca = false; - } - config.YAMLDoc["Systematics"] = YAML::Node(YAML::NodeType::Sequence); - for (auto const &yfn : YAMLFiles) { - for (const auto &[idx, node] : M3::OpenConfig(yf)["Systematics"]) { + for (auto const &yfn : config.inputFiles) { + for (const auto &node : M3OpenConfig(yfn)["Systematics"]) { config.YAMLDoc["Systematics"].push_back(node); } } @@ -156,17 +71,17 @@ ParameterHandlerBase::ParameterHandlerBase( std::vector param_infos; // ETA - read in the systematics. - for (auto const &[idx, node] : config.YAMLDoc["Systematics"]) { - size_t param_id = param_infos.size(); + for (auto const &node : config.YAMLDoc["Systematics"]) { + int param_id = int(param_infos.size()); auto const &pardef = node["Systematic"]; - auto fancy_name = GetFromManager(pardef["Names"]["FancyName"], - __FILE__, __LINE__); - auto prefit = GetFromManager( - pardef["ParameterValues"]["PreFitValue"], __FILE__, __LINE__); + auto fancy_name = + Get(pardef["Names"]["FancyName"], __FILE__, __LINE__); + auto prefit = Get(pardef["ParameterValues"]["PreFitValue"], + __FILE__, __LINE__); - auto error = GetFromManager(pardef["Error"], __FILE__, __LINE__); + auto error = Get(pardef["Error"], __FILE__, __LINE__); if (error <= 0) { MACH3LOG_ERROR("Error for param {}({}) is negative and equal to {}", @@ -175,7 +90,7 @@ ParameterHandlerBase::ParameterHandlerBase( } auto stepscale = - GetFromManager(pardef["StepScale"]["MCMC"], __FILE__, __LINE__); + Get(pardef["StepScale"]["MCMC"], __FILE__, __LINE__); auto bounds = GetBounds(pardef["ParameterBounds"]); @@ -192,41 +107,41 @@ ParameterHandlerBase::ParameterHandlerBase( GetFromManager(pardef["FixParam"], false, __FILE__, __LINE__); if (pardef["SpecialProposal"]) { - special_proposals.push_back(param_id); - EnableSpecialProposal(pardef["SpecialProposal"]); + EnableSpecialProposal(pardef["SpecialProposal"], param_id); } if (pardef["Correlations"]) { - for (auto const &[_, it] : pardef["Correlations"]) { - for (auto const &[key, corr] : it) { - parameter_correlations[param_id][key.as()] = - corr.as(); + for (auto const &corrn : pardef["Correlations"]) { + for (auto const &corr : corrn) { + parameter_correlations[param_id][corr.first.as()] = + corr.second.as(); } } } - param_infos.emplace_back({fancy_name, - fancy_name, - prefit, - error, - stepscale, - {bounds[0], bounds[1]}, - flatprior, - samplenames}); + param_infos.emplace_back(ParamInfo{fancy_name, + fancy_name, + prefit, + error, + stepscale, + {bounds[0], bounds[1]}, + flatprior, + !fixed, + samplenames}); } AddParameters(param_infos); for (auto const &[paramid, correlations] : parameter_correlations) { - SetParameterCorrelations(paramid, correlations); + SetParameterAllCorrelations(paramid, correlations); } - params.covariance = M3::MakePosDef(params.covariance); + params.covariance = M3::MakeMatrixPosDef(params.covariance); Tunes = ParameterTunes(config.YAMLDoc["Systematics"]); MACH3LOG_INFO("Created covariance matrix from files: "); - for (const auto &file : YAMLFile) { + for (const auto &file : config.inputFiles) { MACH3LOG_INFO("{} ", file); } MACH3LOG_INFO("----------------"); @@ -236,29 +151,69 @@ ParameterHandlerBase::ParameterHandlerBase( } // ******************************************** -void ParameterHandlerBase::ConstructPCA(const double eigen_threshold, - int FirstPCAdpar, int LastPCAdpar) { +void ParameterHandlerBase::ConstructPCA(const double threshold, int first, + int last) { // ******************************************** - if (AdaptiveHandler) { + if (settings.use_adaptive) { MACH3LOG_ERROR("Adaption has been enabled and now trying to enable PCA. " "Right now both configuration don't work with each other"); throw MaCh3Exception(__FILE__, __LINE__); } - // Check whether first and last pcadpar are set and if not just PCA everything - if (FirstPCAdpar == -999 || LastPCAdpar == -999) { - if (FirstPCAdpar == -999 && LastPCAdpar == -999) { - FirstPCAdpar = 0; - LastPCAdpar = covMatrix->GetNrows() - 1; + block_indices[0] = first; + block_indices[1] = last; + + int nphysics_parameters = (last - first) + 1; + + double block_trace = + params.covariance.block(first, first, last - first + 1, last - first + 1) + .trace(); + + // a covariance is real symmetric, so self adjoint + Eigen::SelfAdjointEigenSolver EigenSolver( + params.covariance.block(first, first, nphysics_parameters, + nphysics_parameters)); + Eigen::VectorXd eigen_val = EigenSolver.eigenvalues(); + Eigen::MatrixXd eigen_vect = EigenSolver.eigenvectors(); + std::vector> evals; + for (int i = 0; i < eigen_val.size(); ++i) { + evals.emplace_back(i, eigen_val[i]); + } + std::sort( + evals.begin(), evals.end(); + [](std::pair const &l, std::pair const &r) { + return l.second < r.second; + }); + + double evsum = 0; + int northo_parameters = 0; + + for (auto &[i, ev] : evals) { + if (std::fabs(ev) > threshold) { + evsum += std::fabs(ev); + northo_parameters++; } else { - MACH3LOG_ERROR("You must either leave FirstPCAdpar and LastPCAdpar at " - "-999 or set them both to something"); - throw MaCh3Exception(__FILE__, __LINE__); + break; } } - PCAObj.ConstructPCA(params.covariance, FirstPCAdpar, LastPCAdpar, - eigen_threshold); + //rows of this matrix correspond to eigenvectors of the input matrix + // we can go from parameters defined in the orthogonal basis back to the + // 'physics' parameters by RowVectOfOrthoParamVals * pca.ortho_to_physics + pca.ortho_to_physics = + Eigen::MatrixXd::Zero(evals.size(), nphysics_parameters); + + for (size_t i = 0; i < northo_parameters; ++i) { + for (size_t j = 0; j < nphysics_parameters; ++j) { + pca.ortho_to_physics.row(i)[j] = + eigen_vect.col(evals[i].first)[evals[j].first] * + std::sqrt(evals[i].second); + } + } + + MACH3LOG_INFO("Threshold of {} on eigen values, kept {}/{} for a total " + "variance of {}/{}", + threshold, nkept, evals.size(), evsum, block_trace); } // ******************************************** @@ -286,45 +241,39 @@ void ParameterHandlerBase::EnableSpecialProposal(const YAML::Node ¶m, } if (CircEnabled) { - special_proposal.CircularBoundsIndex.push_back(Index); - special_proposal.CircularBoundsValues.push_back( - Get>(param["CircularBounds"], __FILE__, - __LINE__)); - MACH3LOG_INFO( - "Enabling CircularBounds for parameter {} with range [{}, {}]", - GetParFancyName(Index), - special_proposal.CircularBoundsValues.back().first, - special_proposal.CircularBoundsValues.back().second); + auto bounds = Get>(param["CircularBounds"], + __FILE__, __LINE__); // KS: Make sure circular bounds are within physical bounds. If we are // outside of physics bound MCMC will never explore such phase space region - if (special_proposal.CircularBoundsValues.back().first < - _fLowBound.at(Index) || - special_proposal.CircularBoundsValues.back().second > - _fUpBound.at(Index)) { + if (bounds.first < params.lowbound[Index] || + bounds.second > params.upbound[Index]) { MACH3LOG_ERROR("Circular bounds [{}, {}] for parameter {} exceed " "physical bounds [{}, {}]", - special_proposal.CircularBoundsValues.back().first, - special_proposal.CircularBoundsValues.back().second, - GetParFancyName(Index), _fLowBound.at(Index), - _fUpBound.at(Index)); + bounds.first, bounds.second, GetParFancyName(Index), + params.lowbound[Index], params.upbound[Index]); throw MaCh3Exception(__FILE__, __LINE__); } + + special_proposal.circ_bounds.emplace_back( + std::make_tuple(Index, bounds.first, bounds.second)); + + MACH3LOG_INFO( + "Enabling CircularBounds for parameter {} with range [{}, {}]", + GetParFancyName(Index), bounds.first, bounds.second); } if (FlipEnabled) { - special_proposal.FlipParameterIndex.push_back(Index); - special_proposal.FlipParameterPoint.push_back( - Get(param["FlipParameter"], __FILE__, __LINE__)); + special_proposal.flips.emplace_back(std::make_pair( + Index, Get(param["FlipParameter"], __FILE__, __LINE__))); MACH3LOG_INFO("Enabling Flipping for parameter {} with value {}", - GetParFancyName(Index), - special_proposal.FlipParameterPoint.back()); + GetParFancyName(Index), special_proposal.flips.back().second); } if (CircEnabled && FlipEnabled) { - const double fp = special_proposal.FlipParameterPoint.back(); - const double low = special_proposal.CircularBoundsValues.back().first; - const double high = special_proposal.CircularBoundsValues.back().second; + const double fp = special_proposal.flips.back().second; + const double low = std::get<1>(special_proposal.circ_bounds.back()); + const double high = std::get<2>(special_proposal.circ_bounds.back()); if (fp < low || fp > high) { MACH3LOG_ERROR("FlipParameter value {} for parameter {} is outside the " @@ -343,7 +292,7 @@ void ParameterHandlerBase::EnableSpecialProposal(const YAML::Node ¶m, if (min_flip < low || max_flip > high) { MACH3LOG_ERROR("Flipping about point {} for parameter {} would leave " "circular bounds [{}, {}]", - special_proposal.FlipParameterPoint.back(), + special_proposal.flips.back().second, GetParFancyName(Index), low, high); throw MaCh3Exception(__FILE__, __LINE__); } @@ -363,12 +312,12 @@ void ParameterHandlerBase::SetCovMatrix(TMatrixDSym *cov) { if (cov->GetNrows() != params.prefit.size()) { MACH3LOG_ERROR( "Passed covariance matrix size {0}x{0}, but have {1} parameters.", - ov->GetNrows(), params.prefit.size()); + cov->GetNrows(), params.prefit.size()); throw MaCh3Exception(__FILE__, __LINE__); } params.covariance = M3::ROOTToEigen(*cov); - params.inv_covarance = params.covariance.invert(); + params.inv_covariance = params.covariance.inverse(); SetThrowMatrix(cov); } @@ -397,14 +346,17 @@ void ParameterHandlerBase::ThrowParameters() { // First draw a new random_vector Randomize(); - Eigen::VectorXd deltas = steps.l_proposal * throws.random_vector; + throws.values = steps.l_proposal * throws.random_vector; + + steps.proposed = params.prefit + throws.values; for (int i = 0; i < throws.values.size(); ++i) { int tries = 0; - while (((params.prefit[i] + deltas[i]) < params.lowbound) || - ((params.prefit[i] + deltas[i]) >= params.upbound)) { - deltas[i] = steps.l_proposal.col(i) * rng.gaus(rng.e1); + while ((steps.proposed[i] < params.lowbound[i]) || + (steps.proposed[i] >= params.upbound[i])) { + steps.proposed[i] = + params.prefit[i] + (steps.l_proposal.col(i) * rng.gaus(rng.e1))[i]; tries++; if (tries > 10000) { // KS: Since we are multithreading there is danger that those messages @@ -413,14 +365,13 @@ void ParameterHandlerBase::ThrowParameters() { i); MACH3LOG_WARN("Matrix: {}", settings.name); MACH3LOG_WARN("Param: {}", params.name[i]); - MACH3LOG_WARN("Setting _fPropVal: {} to {}", steps.proposed[i], + MACH3LOG_WARN("Setting steps.proposed: {} to {}", steps.proposed[i], params.prefit[i]); MACH3LOG_WARN("I live at {}:{}", __FILE__, __LINE__); - deltas[i] = 0; + steps.proposed[i] = params.prefit[i]; } } - steps.proposed = params.prefit + deltas; steps.current = steps.proposed; } } @@ -432,42 +383,39 @@ void ParameterHandlerBase::RandomConfiguration() { // ************************************* // Have the 1 sigma for each parameter in each covariance class, sweet! // Don't want to change the prior array because that's what determines our - // likelihood Want to change the _fPropVal, _fCurrVal, _fPreFitValue - // _fPreFitValue and the others will already be set - for (int i = 0; i < _fNumPar; ++i) { + // likelihood Want to change the steps.proposed, steps.current, params.prefit + // params.prefit and the others will already be set + + for (int i = 0; i < params.prefit.size(); ++i) { // Check if parameter is fixed first: if so don't randomly throw - if (IsParameterFixed(i)) + if (IsParameterFixed(i)) { continue; + } // Check that the sigma range is larger than the parameter range // If not, throw in the valid parameter range instead - const double paramrange = _fUpBound[i] - _fLowBound[i]; - const double sigma = sqrt((*covMatrix)(i, i)); - double throwrange = sigma; - if (paramrange < sigma) - throwrange = paramrange; + double throwrange = std::min(params.upbound[i] - params.lowbound[i], + std::sqrt(params.covariance(i, i))); - _fPropVal[i] = _fPreFitValue[i] + random_number[0]->Gaus(0, 1) * throwrange; + steps.proposed[i] = params.prefit[i] + rng.gaus(rng.e1) * throwrange; // Try again if we the initial parameter proposal falls outside of the range // of the parameter - int throws = 0; - while (_fPropVal[i] > _fUpBound[i] || _fPropVal[i] < _fLowBound[i]) { - if (throws > 1000) { - MACH3LOG_WARN("Tried {} times to throw parameter {} but failed", throws, - i); - MACH3LOG_WARN("Matrix: {}", matrixName); - MACH3LOG_WARN("Param: {}", _fNames[i]); + int nthrows = 0; + while ((steps.proposed[i] > params.upbound[i]) || + (steps.proposed[i] < params.lowbound[i])) { + if (nthrows > 1000) { + MACH3LOG_WARN("Tried {} times to throw parameter {} but failed", + nthrows, i); + MACH3LOG_WARN("Matrix: {}", settings.name); + MACH3LOG_WARN("Param: {}", params.name[i]); throw MaCh3Exception(__FILE__, __LINE__); } - _fPropVal[i] = - _fPreFitValue[i] + random_number[0]->Gaus(0, 1) * throwrange; - throws++; + steps.proposed[i] = params.prefit[i] + rng.gaus(rng.e1) * throwrange; + nthrows++; } MACH3LOG_INFO("Setting current step in {} param {} = {} from {}", - matrixName, i, _fPropVal[i], _fCurrVal[i]); - _fCurrVal[i] = _fPropVal[i]; + settings.name, i, steps.proposed[i], steps.current[i]); + steps.current[i] = steps.proposed[i]; } - if (pca) - PCAObj.TransferToPCA(); } // ************************************* @@ -479,8 +427,6 @@ void ParameterHandlerBase::SetSingleParameter(const int parNo, steps.proposed[parNo] = parVal; MACH3LOG_DEBUG("Setting {} (parameter {}) to {})", GetParName(parNo), parNo, parVal); - if (pca) - PCAObj.TransferToPCA(); } // ******************************************** @@ -502,8 +448,9 @@ void ParameterHandlerBase::ProposeStep() { // KS: According to Dr Wallace we update using previous not proposed step // this way we do special proposal after adaptive after. // This way we can shortcut and skip rest of proposal - if (!doSpecialStepProposal) + if (!special_proposal.enabled) { return; + } SpecialStepProposal(); } @@ -514,20 +461,26 @@ void ParameterHandlerBase::SpecialStepProposal() { /// @warning KS: Following Asher comment we do "Step->Circular Bounds->Flip" // HW It should now automatically set dcp to be with [-pi, pi] - for (size_t i = 0; i < special_proposal.CircularBoundsIndex.size(); ++i) { - const int index = special_proposal.CircularBoundsIndex[i]; - if (!IsParameterFixed(index)) - CircularParBounds(index, special_proposal.CircularBoundsValues[i].first, - special_proposal.CircularBoundsValues[i].second); + for (auto const &[idx, low, up] : special_proposal.circ_bounds) { + if (IsParameterFixed(idx)) { + continue; + } + if (steps.proposed[idx] > up) { + steps.proposed[idx] = low + std::fmod(steps.proposed[idx] - up, up - low); + } else if (steps.proposed[idx] < low) { + steps.proposed[idx] = up - std::fmod(low - steps.proposed[idx], up - low); + } } // Okay now we've done the standard steps, we can add in our nice flips // hierarchy flip first - for (size_t i = 0; i < special_proposal.FlipParameterIndex.size(); ++i) { - const int index = special_proposal.FlipParameterIndex[i]; - if (!IsParameterFixed(index)) - FlipParameterValue(special_proposal.FlipParameterIndex[i], - special_proposal.FlipParameterPoint[i]); + for (auto const &[idx, pivot] : special_proposal.flips) { + if (IsParameterFixed(idx)) { + continue; + } + if (rng.unif(rng.e1) < 0.5) { + steps.proposed[idx] = 2 * pivot - steps.proposed[idx]; + } } } @@ -537,8 +490,10 @@ void ParameterHandlerBase::SpecialStepProposal() { // Also get a new random number for the randParams void ParameterHandlerBase::Randomize() _noexcept_ { // ************************************************ - throws.random_vector = Eigen::VectorXd::NullaryExpr( - params.prefit.size(), [&](int) { return rng.gaus(rng.e1); }); + throws.random_vector = + Eigen::VectorXd::NullaryExpr(params.prefit.size(), [&](int i) { + return params.isfree[i] ? rng.gaus(rng.e1) : 0; + }); } // ************************************************ @@ -547,22 +502,14 @@ void ParameterHandlerBase::Randomize() _noexcept_ { void ParameterHandlerBase::CorrelateSteps() _noexcept_ { // ************************************************ - if (!pca) { -#ifdef MULTITHREAD -#pragma omp parallel for -#endif - for (int i = 0; i < _fNumPar; ++i) { - if (!IsParameterFixed(i) > 0.) { - _fPropVal[i] = _fCurrVal[i] + - corr_throw[i] * _fGlobalStepScale * _fIndivStepScale[i]; - } - } - // If doing PCA throw uncorrelated in PCA basis (orthogonal basis by - // definition) - } else { - PCAObj.CorrelateSteps(_fIndivStepScale, _fGlobalStepScale, randParams, - corr_throw); - } + Randomize(); + + // array makes the multiplaction by steps.scale component-wise rather than + // vector-ie + steps.proposed = + steps.current + ((steps.l_proposal * throws.random_vector).array() * + steps.scale.array() * steps.global_scale) + .matrix(); } // ******************************************** // Update so that current step becomes the previously proposed step @@ -570,82 +517,17 @@ void ParameterHandlerBase::AcceptStep() _noexcept_ { // ******************************************** steps.current = steps.proposed; - if (AdaptiveHandler) { + if (settings.use_adaptive) { AdaptiveHandler.IncrementAcceptedSteps(); } } -// ************************************* -// HW: This method is a tad hacky but modular arithmetic gives me a headache. -void ParameterHandlerBase::CircularParBounds(const int index, - const double LowBound, - const double UpBound) { - // ************************************* - if (_fPropVal[index] > UpBound) { - _fPropVal[index] = - LowBound + std::fmod(_fPropVal[index] - UpBound, UpBound - LowBound); - } else if (_fPropVal[index] < LowBound) { - _fPropVal[index] = - UpBound - std::fmod(LowBound - _fPropVal[index], UpBound - LowBound); - } -} - -// ************************************* -void ParameterHandlerBase::FlipParameterValue(const int index, - const double FlipPoint) { - // ************************************* - if (random_number[0]->Uniform() < 0.5) { - _fPropVal[index] = 2 * FlipPoint - _fPropVal[index]; - } -} - -// ******************************************** -// Throw the proposed parameter by mag sigma -// Should really just have the user specify this throw by having argument double -void ParameterHandlerBase::ThrowParProp(const double mag) { - // ******************************************** - Randomize(); - if (!pca) { - // Make the correlated throw - M3::MatrixVectorMulti(corr_throw, throwMatrixCholDecomp, randParams, - _fNumPar); - // Number of sigmas we throw - for (int i = 0; i < _fNumPar; i++) { - if (!IsParameterFixed(i) > 0.) - _fPropVal[i] = _fCurrVal[i] + corr_throw[i] * mag; - } - } else { - PCAObj.ThrowParProp(mag, randParams); - } -} -// ******************************************** -// Helper function to throw the current parameter by mag sigmas -// Can study bias in MCMC with this; put different starting parameters -void ParameterHandlerBase::ThrowParCurr(const double mag) { - // ******************************************** - Randomize(); - if (!pca) { - // Get the correlated throw vector - M3::MatrixVectorMulti(corr_throw, throwMatrixCholDecomp, randParams, - _fNumPar); - // The number of sigmas to throw - // Should probably have this as a default parameter input to the function - // instead - for (int i = 0; i < _fNumPar; i++) { - if (!IsParameterFixed(i) > 0.) { - _fCurrVal[i] = corr_throw[i] * mag; - } - } - } else { - PCAObj.ThrowParCurr(mag, randParams); - } -} // ******************************************** // Function to print the prior values void ParameterHandlerBase::PrintNominal() const { // ******************************************** MACH3LOG_INFO("Prior values for {} ParameterHandler:", GetName()); - for (int i = 0; i < params.name.size(); i++) { + for (int i = 0; i < int(params.name.size()); i++) { MACH3LOG_INFO(" {} {} ", GetParFancyName(i), GetParInit(i)); } } @@ -659,7 +541,7 @@ void ParameterHandlerBase::PrintNominalCurrProp() const { "Proposed"); for (int i = 0; i < params.prefit.size(); ++i) { MACH3LOG_INFO("{:<30} {:<10.2f} {:<10.2f} {:<10.2f}", GetParFancyName(i), - params.prefit[i], params.current[i], params.proposed[i]); + params.prefit[i], steps.current[i], steps.proposed[i]); } } @@ -674,17 +556,17 @@ double ParameterHandlerBase::CalcLikelihood() const _noexcept_ { // ******************************************** // filter out parameters with flat priors from the pull auto diff = - params.flatprior.select(Eigen::VectorXd::Zeros(params.prefit.size()), - params.proposed - params.prefit); - return diff.T * params.inv_covariance * diff; + params.flatprior.select(Eigen::VectorXd::Zero(params.prefit.size()), + steps.proposed - params.prefit); + return diff.transpose() * params.inv_covariance * diff; } // ******************************************** int ParameterHandlerBase::CheckBounds() const _noexcept_ { // ******************************************** - return ((params.proposed.array() < params.lowbound.array()) || - (params.proposed.array() >= params.upbound.array())) - .count(); + return int(((steps.proposed.array() < params.lowbound.array()) || + (steps.proposed.array() >= params.upbound.array())) + .count()); } // ******************************************** @@ -708,19 +590,19 @@ void ParameterHandlerBase::SetParameters(const std::vector &pars) { steps.proposed = params.prefit; // If not empty, set the parameters to the specified } else { - if (pars.size() != size_t(_fNumPar)) { + if (pars.size() != size_t(steps.proposed.size())) { MACH3LOG_ERROR("Parameter arrays of incompatible size! Not changing " "parameters! {} has size {} but was expecting {}", - matrixName, pars.size(), _fNumPar); + settings.name, pars.size(), steps.proposed.size()); throw MaCh3Exception(__FILE__, __LINE__); } - for (int i = 0; i < pars.size(); i++) { + for (int i = 0; i < int(pars.size()); i++) { // Make sure that you are actually passing a number to set the parameter // to if (std::isnan(pars[i])) { MACH3LOG_ERROR("Trying to set parameter value to a nan for parameter " "{} in matrix {}. This will not go well!", - GetParName(i), matrixName); + GetParName(i), settings.name); throw MaCh3Exception(__FILE__, __LINE__); } else { steps.proposed[i] = pars[i]; @@ -733,25 +615,25 @@ void ParameterHandlerBase::SetParameters(const std::vector &pars) { void ParameterHandlerBase::SetBranches(TTree &tree, bool SaveProposal) { // ******************************************** // loop over parameters and set a branch - for (int i = 0; i < params.name.size(); ++i) { - tree.Branch(params.name[i].c_str(), params.current.data() + i, + for (int i = 0; i < int(params.name.size()); ++i) { + tree.Branch(params.name[i].c_str(), steps.current.data() + i, Form("%s/D", params.name[i].c_str())); } - // When running PCA, also save PCA parameters - if (pca) { - PCAObj.SetBranches(tree, SaveProposal, _fNames); - } + // // When running PCA, also save PCA parameters + // if (pca.enabled) { + // PCAObj.SetBranches(tree, SaveProposal, _fNames); + // } if (SaveProposal) { // loop over parameters and set a branch - for (int i = 0; i < params.name.size(); ++i) { + for (int i = 0; i < int(params.name.size()); ++i) { tree.Branch(Form("%s_Prop", params.name[i].c_str()), - params.proposed.data() + i, + steps.proposed.data() + i, Form("%s_Prop/D", params.name[i].c_str())); } } - if (use_adaptive && AdaptiveHandler.GetUseRobbinsMonro()) { + if (settings.use_adaptive && AdaptiveHandler.GetUseRobbinsMonro()) { tree.Branch(Form("GlobalStepScale_%s", GetName().c_str()), - &_fGlobalStepScale, + &steps.global_scale, Form("GlobalStepScale_%s/D", GetName().c_str())); } } @@ -767,7 +649,7 @@ void ParameterHandlerBase::SetStepScale(const double scale, if (verbose) { MACH3LOG_INFO("{} setStepScale() = {}", GetName(), scale); - const double SuggestedScale = 2.38 / std::sqrt(_fNumPar); + const double SuggestedScale = 2.38 / std::sqrt(steps.scale.size()); if ((std::fabs(scale - SuggestedScale) / SuggestedScale) > 1) { MACH3LOG_WARN( "Defined Global StepScale is {}, while suggested suggested {}", scale, @@ -785,19 +667,19 @@ int ParameterHandlerBase::GetParIndex(const std::string &name) const { MACH3LOG_ERROR("No parameter with the name {} exists.", name); throw MaCh3Exception(__FILE__, __LINE__); } - return std::distance(params.name.begin(), idx); + return int(std::distance(params.name.begin(), idx)); } // ******************************************** void ParameterHandlerBase::SetFixAllParameters() { // ******************************************** - params.fixed = Eigen::VectorXi::Ones(params.prefit.size()); + params.isfree = Eigen::VectorXi::Zero(params.prefit.size()); } // ******************************************** void ParameterHandlerBase::SetFixParameter(const int i) { // ******************************************** - params.fixed[i] = 1; + params.isfree[i] = 0; } // ******************************************** @@ -809,13 +691,13 @@ void ParameterHandlerBase::SetFixParameter(const std::string &name) { // ******************************************** void ParameterHandlerBase::SetFreeAllParameters() { // ******************************************** - params.fixed = Eigen::VectorXi::Zeros(params.prefit.size()); + params.isfree = Eigen::VectorXi::Ones(params.prefit.size()); } // ******************************************** void ParameterHandlerBase::SetFreeParameter(const int i) { // ******************************************** - params.fixed[i] = 0; + params.isfree[i] = 1; } // ******************************************** @@ -831,13 +713,13 @@ void ParameterHandlerBase::ToggleFixAllParameters() { } // ******************************************** -void ParameterHandlerBase::ToggleFixParameter(const int i) { +void ParameterHandlerBase::ToggleFixParameter(const int) { // ******************************************** throw std::runtime_error("Don't Toggle parameters"); } // ******************************************** -void ParameterHandlerBase::ToggleFixParameter(const std::string &name) { +void ParameterHandlerBase::ToggleFixParameter(const std::string &) { // ******************************************** throw std::runtime_error("Don't Toggle parameters"); } @@ -874,15 +756,15 @@ void ParameterHandlerBase::SetFlatPrior(const int i, const bool eL) { void ParameterHandlerBase::SetIndivStepScale( const std::vector &stepscale) { // ******************************************** - if (int(stepscale.size()) != params.stepscale.size()) { + if (int(steps.scale.size()) != steps.scale.size()) { MACH3LOG_WARN( "Stepscale vector not equal to number of parameters. Quitting.."); - MACH3LOG_WARN("Size of argument vector: {}", stepscale.size()); - MACH3LOG_WARN("Expected size: {}", params.stepscale.size()); + MACH3LOG_WARN("Size of argument vector: {}", steps.scale.size()); + MACH3LOG_WARN("Expected size: {}", steps.scale.size()); return; } - params.stepscale = M3::StdVectorToEigen(stepscale); + steps.scale = M3::StdVectorToEigen(stepscale); PrintIndivStepScale(); } @@ -890,10 +772,11 @@ void ParameterHandlerBase::SetIndivStepScale( void ParameterHandlerBase::PrintIndivStepScale() const { // ******************************************** MACH3LOG_INFO("============================================================"); - MACH3LOG_INFO("{:<{}} | {:<11}", "Parameter:", PrintLength, "Step scale:"); - for (int iParam = 0; iParam < params.fancyname.size(); iParam++) { - MACH3LOG_INFO("{:<{}} | {:<11}", params.fancyname[iParam].c_str(), - PrintLength, params.stepscale[iParam]); + MACH3LOG_INFO("{:<{}} | {:<11}", "Parameter:", settings.PrintLength, + "Step scale:"); + for (int iParam = 0; iParam < int(params.fancy_name.size()); iParam++) { + MACH3LOG_INFO("{:<{}} | {:<11}", params.fancy_name[iParam].c_str(), + settings.PrintLength, steps.scale[iParam]); } MACH3LOG_INFO("============================================================"); } @@ -902,7 +785,7 @@ void ParameterHandlerBase::PrintIndivStepScale() const { void ParameterHandlerBase::ResetIndivStepScale() { // ******************************************** steps.global_scale = 1.0; - SetIndivStepScale(std::vector(1, params.stepscale.size())); + SetIndivStepScale(std::vector(steps.scale.size(), 1.0)); } // ******************************************** @@ -917,20 +800,30 @@ void ParameterHandlerBase::SetThrowMatrix(TMatrixDSym *cov) { throw MaCh3Exception(__FILE__, __LINE__); } - if (params.l_proposal.rows() != cov->GetNrows()) { + if (steps.l_proposal.rows() != cov->GetNrows()) { MACH3LOG_ERROR("Matrix given for throw Matrix is not the same size as the " "proposal matrix stored in object!"); - MACH3LOG_ERROR("Stored proposal matrix size: {}", params.l_proposal.rows()); + MACH3LOG_ERROR("Stored proposal matrix size: {}", steps.l_proposal.rows()); MACH3LOG_ERROR("Given matrix size: {}", cov->GetNrows()); throw MaCh3Exception(__FILE__, __LINE__); } - if (use_adaptive && AdaptiveHandler.AdaptionUpdate()) { - M3::MakeClosestPosDef(cov); - params.l_proposal = Eigen::LLT(M3::ROOTToEigen(cov)).L(); + if (settings.use_adaptive && AdaptiveHandler.AdaptionUpdate()) { + M3::MakeMatrixClosestPosDef(cov); + Eigen::LLT lltproposal(M3::ROOTToEigen(*cov)); + if (lltproposal.info() != Eigen::Success) { + MACH3LOG_ERROR("Failed to LLT decompose proposal matrix"); + throw MaCh3Exception(__FILE__, __LINE__); + } + steps.l_proposal = lltproposal.matrixL(); } else { - params.l_proposal = - Eigen::LLT(M3::MakePosDef(M3::ROOTToEigen(*cov))).L(); + Eigen::LLT lltproposal( + M3::MakeMatrixPosDef(M3::ROOTToEigen(*cov))); + if (lltproposal.info() != Eigen::Success) { + MACH3LOG_ERROR("Failed to LLT decompose proposal matrix"); + throw MaCh3Exception(__FILE__, __LINE__); + } + steps.l_proposal = lltproposal.matrixL(); } } @@ -944,12 +837,12 @@ void ParameterHandlerBase::UpdateThrowMatrix(TMatrixDSym *cov) { // HW : Here be adaption void ParameterHandlerBase::InitialiseAdaption(const YAML::Node &adapt_manager) { // ******************************************** - if (PCAObj) { + if (pca.enabled) { MACH3LOG_ERROR("PCA has been enabled and now trying to enable Adaption. " "Right now both configuration don't work with each other"); throw MaCh3Exception(__FILE__, __LINE__); } - if (AdaptiveHandler) { + if (settings.use_adaptive) { MACH3LOG_ERROR("Adaptive Handler has already been initialise can't do it " "again so skipping."); return; @@ -957,25 +850,26 @@ void ParameterHandlerBase::InitialiseAdaption(const YAML::Node &adapt_manager) { // Now we read the general settings [these SHOULD be common across all // matrices!] - bool success = AdaptiveHandler.InitFromConfig(adapt_manager, matrixName, - &_fCurrVal, &_fError); - if (!success) { - return; - } + // bool success = AdaptiveHandler.InitFromConfig(adapt_manager, settings.name, + // &steps.current, + // ¶ms.error); + // if (!success) { + // return; + // } AdaptiveHandler.Print(); // Next let"s check for external matrices // We"re going to grab this info from the YAML manager if (!GetFromManager(adapt_manager["AdaptionOptions"]["Covariance"] - [matrixName]["UseExternalMatrix"], + [settings.name]["UseExternalMatrix"], false, __FILE__, __LINE__)) { MACH3LOG_WARN( "Not using external matrix for {}, initialising adaption from scratch", - matrixName); + settings.name); // If we don't have a covariance matrix to start from for adaptive tune we // need to make one! - use_adaptive = true; + settings.use_adaptive = true; AdaptiveHandler.CheckMatrixValidityForAdaption(GetCovMatrix()); AdaptiveHandler.CreateNewAdaptiveCovariance(); return; @@ -983,21 +877,21 @@ void ParameterHandlerBase::InitialiseAdaption(const YAML::Node &adapt_manager) { // Finally, we accept that we want to read the matrix from a file! auto external_file_name = GetFromManager( - adapt_manager["AdaptionOptions"]["Covariance"][matrixName] + adapt_manager["AdaptionOptions"]["Covariance"][settings.name] ["ExternalMatrixFileName"], "", __FILE__, __LINE__); auto external_matrix_name = GetFromManager( - adapt_manager["AdaptionOptions"]["Covariance"][matrixName] - ["ExternalMatrixName"], + adapt_manager["AdaptionOptions"]["Covariance"][settings.name] + ["Externalsettings.name"], "", __FILE__, __LINE__); auto external_mean_name = GetFromManager( - adapt_manager["AdaptionOptions"]["Covariance"][matrixName] + adapt_manager["AdaptionOptions"]["Covariance"][settings.name] ["ExternalMeansName"], "", __FILE__, __LINE__); - AdaptiveHandler.SetThrowMatrixFromFile(external_file_name, - external_matrix_name, - external_mean_name, use_adaptive); + AdaptiveHandler.SetThrowMatrixFromFile( + external_file_name, external_matrix_name, external_mean_name, + settings.use_adaptive); SetThrowMatrix(AdaptiveHandler.GetAdaptiveCovariance()); ResetIndivStepScale(); @@ -1058,7 +952,7 @@ void ParameterHandlerBase::SaveUpdatedMatrixConfig( if (!config.YAMLDoc) { MACH3LOG_CRITICAL("Yaml node hasn't been initialised for matrix {}, " "something is not right", - matrixName); + settings.name); MACH3LOG_CRITICAL("I am not throwing error but should be investigated"); return; } @@ -1069,7 +963,7 @@ void ParameterHandlerBase::SaveUpdatedMatrixConfig( for (YAML::Node param : copyNode["Systematics"]) { // KS: Feel free to update it, if you need updated prefit value etc param["Systematic"]["StepScale"]["MCMC"] = - MaCh3Utils::FormatDouble(params.stepscale[i], 4); + MaCh3Utils::FormatDouble(steps.scale[i], 4); i++; } // Save the modified node to a file @@ -1101,9 +995,9 @@ bool ParameterHandlerBase::AppliesToSample( bool Applies = false; - for (size_t i = 0; i < _fSampleNames[SystIndex].size(); i++) { + for (size_t i = 0; i < params.samples[SystIndex].size(); i++) { // Convert to low case to not be case sensitive - std::string pattern = _fSampleNames[SystIndex][i]; + std::string pattern = params.samples[SystIndex][i]; std::transform(pattern.begin(), pattern.end(), pattern.begin(), ::tolower); // Replace '*' in the pattern with '.*' for regex matching @@ -1127,7 +1021,7 @@ bool ParameterHandlerBase::AppliesToSample( // Set proposed parameter values vector to be base on tune values void ParameterHandlerBase::SetTune(const std::string &TuneName) { // ******************************************** - SetParameters(Tunes->GetTune(TuneName)); + SetParameters(Tunes.GetTune(TuneName)); } // ************************************* diff --git a/Parameters/ParameterHandlerBase.h b/Parameters/ParameterHandlerBase.h index a36cb0d8c..f6c0a1e23 100644 --- a/Parameters/ParameterHandlerBase.h +++ b/Parameters/ParameterHandlerBase.h @@ -7,7 +7,12 @@ #include "Parameters/ParameterHandlerUtils.h" #include "Parameters/ParameterTunes.h" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Walloca" #include "Eigen/Dense" +#pragma GCC diagnostic pop + +#include /// @brief Base class responsible for handling of systematic error parameters. /// Capable of using PCA or using adaptive throw matrix @@ -22,17 +27,22 @@ class ParameterHandlerBase { std::string name, fancy_name; double prefit, error, stepscale; std::array bounds; - bool flatprior, fixed; + bool flatprior, isfree; std::vector affected_samples; }; void AddParameters(std::vector const ¶ms); void AddParameter(ParamInfo const ¶m) { AddParameters({ - params, + param, }); } + void SetParameterCorrelation(int pidi, int pidj, double corr); + + void SetParameterAllCorrelations( + int paramid, std::map const &correlations); + /// @brief ETA - constructor for a YAML file /// @param YAMLFile A vector of strings representing the YAML files used for /// initialisation of matrix @@ -44,8 +54,10 @@ class ParameterHandlerBase { MakeFromYAML(const std::vector &YAMLFiles, std::string name, double threshold = -1, int FirstPCAdpar = -999, int LastPCAdpar = -999) { - return ParameterHandlerBase(YAMLFiles, name, threshold, FirstPCAdpar, - LastPCAdpar); + (void)threshold; + (void)FirstPCAdpar; + (void)LastPCAdpar; + return ParameterHandlerBase(YAMLFiles, name); } /// @brief "Usual" constructors from root file /// @param name Matrix name @@ -53,8 +65,10 @@ class ParameterHandlerBase { static ParameterHandlerBase MakeFromTMatrix(std::string name, std::string file, double threshold = -1, int FirstPCAdpar = -999, int LastPCAdpar = -999) { - return ParameterHandlerBase(name, file, threshold, FirstPCAdpar, - LastPCAdpar); + (void)threshold; + (void)FirstPCAdpar; + (void)LastPCAdpar; + return ParameterHandlerBase(name, file); } /// @brief Destructor @@ -100,7 +114,7 @@ class ParameterHandlerBase { /// @param i Parameter index /// @param val new value which will be set /// @ingroup ParameterHandlerSetters - void SetParProp(const int i, const double val) { params.proposed[i] = val; } + void SetParProp(const int i, const double val) { steps.proposed[i] = val; } /// @brief Set parameter values using vector, it has to have same size as /// covariance class /// @param pars Vector holding new values for every parameter @@ -202,10 +216,12 @@ class ParameterHandlerBase { /// @brief Return covariance matrix /// @ingroup ParameterHandlerGetters - TMatrixDSym *GetCovMatrix() const { return &root_copies.covariance; } + TMatrixDSym const *GetCovMatrix() const { return &root_copies.covariance; } /// @brief Return inverted covariance matrix /// @ingroup ParameterHandlerGetters - TMatrixDSym *GetInvCovMatrix() const { return &root_copies.inv_covariance; } + TMatrixDSym const *GetInvCovMatrix() const { + return &root_copies.inv_covariance; + } /// @brief Return inverted covariance matrix /// @ingroup ParameterHandlerGetters double GetInvCovMatrix(const int i, const int j) const { @@ -215,16 +231,16 @@ class ParameterHandlerBase { /// @brief Return correlated throws /// @param i Parameter index /// @ingroup ParameterHandlerGetters - double GetCorrThrows(const int i) const { return corr_throw[i]; } + double GetCorrThrows(const int i) const { return throws.values[i]; } /// @brief Get if param has flat prior or not /// @param i Parameter index /// @ingroup ParameterHandlerGetters - inline bool GetFlatPrior(const int i) const { return params.flatprior[i]; } + bool GetFlatPrior(const int i) const { return params.flatprior[i]; } /// @brief Get name of covariance /// @ingroup ParameterHandlerGetters - std::string GetName() const { return params.matrix_name; } + std::string GetName() const { return settings.name; } /// @brief Get name of parameter /// @param i Parameter index /// @ingroup ParameterHandlerGetters @@ -237,21 +253,23 @@ class ParameterHandlerBase { /// @brief Get fancy name of the Parameter /// @param i Parameter index /// @ingroup ParameterHandlerGetters - std::string GetParFancyName(const int i) const { return params.fancyname[i]; } + std::string GetParFancyName(const int i) const { + return params.fancy_name[i]; + } /// @brief Get name of input file /// @ingroup ParameterHandlerGetters - std::string GetInputFile() const { return config.inputFile; } + std::string GetInputFile() const { return config.inputFiles[0]; } /// @brief Get diagonal error for ith parameter /// @param i Parameter index /// @ingroup ParameterHandlerGetters - inline double GetDiagonalError(const int i) const { + double GetDiagonalError(const int i) const { return std::sqrt(params.covariance(i, i)); } /// @brief Get the error for the ith parameter /// @param i Parameter index /// @ingroup ParameterHandlerGetters - inline double GetError(const int i) const { return params.error[i]; } + double GetError(const int i) const { return params.error[i]; } /// @brief Adaptive Step Tuning Stuff void ResetIndivStepScale(); @@ -276,7 +294,7 @@ class ParameterHandlerBase { /// @brief Set number of MCMC step, when running adaptive MCMC it is updated /// with given frequency. We need number of steps to determine frequency. /// @ingroup ParameterHandlerSetters - inline void SetNumberOfSteps(const int nsteps) { + void SetNumberOfSteps(const int nsteps) { AdaptiveHandler.SetTotalSteps(nsteps); if (AdaptiveHandler.AdaptionUpdate()) { ResetIndivStepScale(); @@ -285,11 +303,11 @@ class ParameterHandlerBase { /// @brief Get matrix used for step proposal /// @ingroup ParameterHandlerGetters - inline TMatrixDSym *GetThrowMatrix() const { return &root_copies.proposal; } + TMatrixDSym const *GetThrowMatrix() const { return &root_copies.proposal; } /// @brief Get matrix used for step proposal /// @ingroup ParameterHandlerGetters double GetThrowMatrix(const int i, const int j) const { - return params.l_proposal(i, j); + return steps.l_proposal(i, j); } /// @brief KS: Convert covariance matrix to correlation matrix and return TH2D @@ -304,7 +322,7 @@ class ParameterHandlerBase { /// @brief Get total number of parameters /// @ingroup ParameterHandlerGetters - inline int GetNumParams() const { return params.prefit.size(); } + int GetNumParams() const { return int(params.prefit.size()); } /// @brief Get the pre-fit values of the parameters. /// @ingroup ParameterHandlerGetters std::vector GetPreFitValues() const { @@ -313,48 +331,41 @@ class ParameterHandlerBase { /// @brief Get vector of all proposed parameter values /// @ingroup ParameterHandlerGetters std::vector GetProposed() const { - return M3::EigenToStdVector(params.proposed); + return M3::EigenToStdVector(steps.proposed); } /// @brief Get proposed parameter value /// @param i Parameter index /// @ingroup ParameterHandlerGetters - inline double GetParProp(const int i) const { return params.proposed[i]; } + double GetParProp(const int i) const { return steps.proposed[i]; } /// @brief Get current parameter value /// @param i Parameter index /// @ingroup ParameterHandlerGetters - inline double GetParCurr(const int i) const { return params.current[i]; } + double GetParCurr(const int i) const { return steps.current[i]; } /// @brief Get prior parameter value /// @param i Parameter index /// @ingroup ParameterHandlerGetters - inline double GetParInit(const int i) const { return params.prefit[i]; } + double GetParInit(const int i) const { return params.prefit[i]; } /// @brief Get upper parameter bound in which it is physically valid /// @param i Parameter index /// @ingroup ParameterHandlerGetters - inline double GetUpperBound(const int i) const { return params.upbound[i]; } + double GetUpperBound(const int i) const { return params.upbound[i]; } /// @brief Get lower parameter bound in which it is physically valid /// @param i Parameter index /// @ingroup ParameterHandlerGetters - inline double GetLowerBound(const int i) const { return params.lowbound[i]; } + double GetLowerBound(const int i) const { return params.lowbound[i]; } /// @brief Get individual step scale for selected parameter /// @param ParameterIndex Parameter index /// @ingroup ParameterHandlerGetters - inline double GetIndivStepScale(const int i) const { - return params.stepscale[i]; - } + double GetIndivStepScale(const int i) const { return steps.scale[i]; } /// @brief Get global step scale for covariance object /// @ingroup ParameterHandlerGetters - inline double GetGlobalStepScale() const { return settings.GlobalStepScale; } + double GetGlobalStepScale() const { return steps.global_scale; } /// @brief Get number of params which will be different depending if using /// Eigen decomposition or not /// @ingroup ParameterHandlerGetters - inline int GetNParameters() const { - if (pca) - return PCAObj->GetNumberPCAedParameters(); - else - return GetNumParams(); - } + int GetNParameters() const { return GetNumParams(); } /// @brief Print prior value for every parameter void PrintNominal() const; @@ -409,7 +420,7 @@ class ParameterHandlerBase { void ToggleFixParameter(const std::string &name); /// @brief Is parameter fixed or not /// @param i Parameter index - bool IsParameterFixed(const int i) const { return params.fixed[i]; } + bool IsParameterFixed(const int i) const { return !params.isfree[i]; } /// @brief Is parameter fixed or not /// @param name Name of parameter you want to check if is fixed @@ -427,7 +438,7 @@ class ParameterHandlerBase { int LastPCAdpar); /// @brief is PCA, can use to query e.g. LLH scans - inline bool IsPCA() const { return pca; } + bool IsPCA() const { return pca.enabled; } /// @brief Getter to return a copy of the YAML node /// @ingroup ParameterHandlerGetters @@ -435,8 +446,8 @@ class ParameterHandlerBase { /// @brief Get pointer for AdaptiveHandler /// @ingroup ParameterHandlerGetters - inline adaptive_mcmc::AdaptiveMCMCHandler &GetAdaptiveHandler() const { - if (!use_adaptive) { + adaptive_mcmc::AdaptiveMCMCHandler const &GetAdaptiveHandler() const { + if (!settings.use_adaptive) { MACH3LOG_ERROR("Am not running in Adaptive mode"); throw MaCh3Exception(__FILE__, __LINE__); } @@ -448,15 +459,6 @@ class ParameterHandlerBase { /// @ingroup ParameterHandlerSetters void SetTune(const std::string &TuneName); - /// @brief Get pointer for PCAHandler - inline PCAHandler &GetPCAHandler() const { - if (!pca) { - MACH3LOG_ERROR("Am not running in PCA mode"); - throw MaCh3Exception(__FILE__, __LINE__); - } - return PCAObj; - } - /// @brief Matches branches in a TTree to parameters in a systematic handler. /// /// @param PosteriorFile Pointer to the ROOT TTree from MaCh3 fit. @@ -473,6 +475,11 @@ class ParameterHandlerBase { protected: ParameterHandlerBase(); + /// @brief "Usual" constructors from root file + /// @param name Matrix name + /// @param file Path to matrix root file + ParameterHandlerBase(std::string const &name, std::string const &file); + /// @brief ETA - constructor for a YAML file /// @param YAMLFile A vector of strings representing the YAML files used for /// initialisation of matrix @@ -481,14 +488,7 @@ class ParameterHandlerBase { /// @param FirstPCAdpar First PCA parameter that will be decomposed. /// @param LastPCAdpar First PCA parameter that will be decomposed. ParameterHandlerBase(const std::vector &YAMLFiles, - std::string name, double threshold = -1, - int FirstPCAdpar = -999, int LastPCAdpar = -999); - /// @brief "Usual" constructors from root file - /// @param name Matrix name - /// @param file Path to matrix root file - ParameterHandlerBase(std::string name, std::string file, - double threshold = -1, int FirstPCAdpar = -999, - int LastPCAdpar = -999); + std::string const &name); /// @brief sets throw matrix from a file /// @param matrix_file_name name of file matrix lives in @@ -526,7 +526,7 @@ class ParameterHandlerBase { struct { bool enabled; std::array block_indices; - Eigen::MatrixXd rotation; + Eigen::MatrixXd ortho_to_physics; } pca; struct { @@ -541,20 +541,18 @@ class ParameterHandlerBase { struct { bool enabled; - /// Indices of parameters with flip symmetry - std::vector FlipParameterIndex; - /// Central points around which parameters are flipped - std::vector FlipParameterPoint; - /// Indices of parameters with circular bounds - std::vector CircularBoundsIndex; - /// Circular bounds for each parameter (lower, upper) - std::vector> CircularBoundsValues; + + // param_index, flip pivot value + std::vector> flips; + // param_index, low bound, up bound + std::vector> circ_bounds; + } special_proposal; struct { std::vector name, fancy_name; Eigen::VectorXd prefit, error, lowbound, upbound; - Eigen::VectorXi flatprior, fixed; + Eigen::VectorXi flatprior, isfree; std::vector> samples; Eigen::MatrixXd covariance; @@ -564,6 +562,7 @@ class ParameterHandlerBase { struct { std::ranlux48 e1; std::normal_distribution gaus; + std::uniform_real_distribution unif; } rng; // to retain external interface where possible @@ -586,7 +585,7 @@ class ParameterHandlerBase { struct { /// The input root file we read in - std::string inputFile; + std::vector inputFiles; /// Stores config describing systematics YAML::Node YAMLDoc; } config; diff --git a/Parameters/ParameterHandlerUtils.h b/Parameters/ParameterHandlerUtils.h index aa2916e1c..cb6b1e9cf 100644 --- a/Parameters/ParameterHandlerUtils.h +++ b/Parameters/ParameterHandlerUtils.h @@ -4,6 +4,15 @@ #include "Parameters/ParameterStructs.h" _MaCh3_Safe_Include_Start_ //{ + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Walloca" +#pragma GCC diagnostic ignored "-Wold-style-cast" +#pragma GCC diagnostic ignored "-Wuseless-cast" +#pragma GCC diagnostic ignored "-Wconversion" +#include "Eigen/Dense" +#pragma GCC diagnostic pop + // ROOT includes #include "TDecompChol.h" #include "TDecompSVD.h" @@ -612,10 +621,10 @@ _MaCh3_Safe_Include_Start_ //{ constexpr int MaxAttempts = 1e5; for (int iAttempt = 0; iAttempt < MaxAttempts; iAttempt++) { - if (Eigen::LLT(cov).info() == Eigen::Success) { + if (Eigen::LLT(cov).info() == Eigen::Success) { return cov; } else { - cov.diagonal() += 1E-9; + cov.diagonal().array() += 1E-9; } } @@ -632,7 +641,7 @@ _MaCh3_Safe_Include_Start_ //{ // ||.||_frob // Where ||X||_frob=sqrt[sum_ij(x_ij^2)] (basically just turns an n,n matrix // into vector in n^2 space then does Euclidean norm) - void MakeClosestPosDef(TMatrixDSym * cov) { + inline void MakeMatrixClosestPosDef(TMatrixDSym * cov) { // ******************************************** // Want to get cov' = (cov_sym+cov_polar)/2 // cov_sym=(cov+cov^T)/2 @@ -651,8 +660,8 @@ _MaCh3_Safe_Include_Start_ //{ TDecompSVD cov_sym_svd = TDecompSVD(cov_sym); if (!cov_sym_svd.Decompose()) { MACH3LOG_WARN( - "Cannot do SVD on input matrix, trying MakePosDef() first!"); - MakePosDef(&cov_sym); + "Cannot do SVD on input matrix, trying MakeMatrixPosDef() first!"); + MakeMatrixPosDef(&cov_sym); } TMatrixD cov_sym_v = cov_sym_svd.GetV(); @@ -683,27 +692,29 @@ _MaCh3_Safe_Include_Start_ //{ *cov = cov_closest_approx; // Now can just add a makeposdef! - MakePosDef(cov); + MakeMatrixPosDef(cov); } // ******************************************** // KS: Convert covariance matrix to correlation matrix and return TH2D which // can be used for fancy plotting - TH2D *GetCorrelationMatrix(Eigen::MatrixXd const &covariance) { + inline TH2D *GetCorrelationMatrix(std::string const &name, + std::vector const &parnames, + Eigen::MatrixXd const &covariance) { // ******************************************** - TH2D *hMatrix = - new TH2D(GetName().c_str(), GetName().c_str(), covariance.rows(), 0.0, - covariance.rows(), covariance.rows(), 0.0, covariance.rows()); + int nrows = int(covariance.rows()); + TH2D *hMatrix = new TH2D(name.c_str(), name.c_str(), nrows, 0.0, nrows, + nrows, 0.0, nrows); hMatrix->SetDirectory(nullptr); - for (int i = 0; i < covariance.rows(); i++) { + for (int i = 0; i < nrows; i++) { hMatrix->SetBinContent(i + 1, i + 1, 1.); - hMatrix->GetXaxis()->SetBinLabel(i + 1, GetParFancyName(i).c_str()); - hMatrix->GetYaxis()->SetBinLabel(i + 1, GetParFancyName(i).c_str()); + hMatrix->GetXaxis()->SetBinLabel(i + 1, parnames[i].c_str()); + hMatrix->GetYaxis()->SetBinLabel(i + 1, parnames[i].c_str()); } auto sqrt_diag = covariance.diagonal().cwiseSqrt(); - for (int i = 0; i < covariance.rows(); i++) { + for (int i = 0; i < nrows; i++) { for (int j = 0; j <= i; j++) { const double Corr = covariance(i, j) / sqrt_diag(i) * sqrt_diag(j); hMatrix->SetBinContent(i + 1, j + 1, Corr); @@ -713,7 +724,7 @@ _MaCh3_Safe_Include_Start_ //{ return hMatrix; } - Eigen::MatrixXd ROOTToEigen(TMatrixDSym const &mat) { + inline Eigen::MatrixXd ROOTToEigen(TMatrixDSym const &mat) { auto emat = Eigen::MatrixXd(mat.GetNrows(), mat.GetNcols()); for (int ri = 0; ri < mat.GetNrows(); ++ri) { for (int ci = 0; ci < mat.GetNcols(); ++ci) { @@ -723,8 +734,8 @@ _MaCh3_Safe_Include_Start_ //{ return emat; } - void EigenToROOT(Eigen::MatrixXd const &emat, TMatrixDSym &mat) { - mat.ResizeTo(emat.rows(), emat.cols()); + inline void EigenToROOT(Eigen::MatrixXd const &emat, TMatrixDSym &mat) { + mat.ResizeTo(int(emat.rows()), int(emat.cols())); for (int ri = 0; ri < emat.rows(); ++ri) { for (int ci = 0; ci < emat.cols(); ++ci) { mat(ri, ci) = emat(ri, ci); @@ -732,21 +743,21 @@ _MaCh3_Safe_Include_Start_ //{ } } - std::vector EigenToStdVector(Eigen::VectorXd const &evec) { + inline std::vector EigenToStdVector(Eigen::VectorXd const &evec) { return std::vector(evec.begin(), evec.end()); } - Eigen::VectorXd StdVectorToEigen(std::vector const &svec) { + inline Eigen::VectorXd StdVectorToEigen(std::vector const &svec) { Eigen::VectorXd evec(svec.size()); - for (size_t i = 0; i < svec; ++i) { + for (size_t i = 0; i < svec.size(); ++i) { evec(i) = svec[i]; } return evec; } - void EnsureNoOutOfBlockCorrelations(Eigen::MatrixXd const &CovMatrix, - std::array block_definition, - double threshold = 1E-6) { + inline void EnsureNoOutOfBlockCorrelations( + Eigen::MatrixXd const &CovMatrix, std::array block_definition, + double correlation_threshold = 1E-6) { for (int i = block_definition[0]; i <= block_definition[1]; ++i) { for (int j = 0; j < CovMatrix.rows(); ++j) { @@ -756,8 +767,10 @@ _MaCh3_Safe_Include_Start_ //{ continue; } - if (std::fabs(CovMatrix(i, j)) > correlation_threshold) { - found_significant_correlation = true; + double corr_val = + CovMatrix(i, j) / std::sqrt(CovMatrix(i, i) * CovMatrix(j, j)); + + if (std::fabs(corr_val) > correlation_threshold) { MACH3LOG_ERROR("Significant correlation detected between decomposed " "parameter '{}' " "and undecomposed parameter '{}': {:.6e}", diff --git a/Parameters/ParameterList.cpp b/Parameters/ParameterList.cpp new file mode 100644 index 000000000..8512ff879 --- /dev/null +++ b/Parameters/ParameterList.cpp @@ -0,0 +1,478 @@ +#include "Parameters/Parameterlist.h" + +#include "Parameters/PCA.h" +#include "Parameters/ParameterHandlerUtils.h" + +void ParameterList::AddParameters(std::vector const &new_params) { + + for (auto const &p : new_params) { + params.name.push_back(p.name); + params.fancy_name.push_back(p.fancy_name); + params.samples.push_back(p.affected_samples); + params.flip_pivot.push_back(p.flip_pivot); + params.circ_bounds.push_back(p.circ_bounds); + } + + size_t new_block_start = params.prefit.size(); + + params.prefit.conservativeResize(params.name.size()); + params.error.conservativeResize(params.name.size()); + params.lowbound.conservativeResize(params.name.size()); + params.upbound.conservativeResize(params.name.size()); + params.stepscale.conservativeResize(params.name.size()); + params.flatprior.conservativeResize(params.name.size()); + params.isfree.conservativeResize(params.name.size()); + + params.covariance.conservativeResize(params.name.size(), params.name.size()); + + for (size_t i = 0; i < new_params.size(); ++i) { + params.prefit[new_block_start + i] = new_params[i].prefit; + params.error[new_block_start + i] = new_params[i].error; + params.lowbound[new_block_start + i] = new_params[i].bounds[0]; + params.upbound[new_block_start + i] = new_params[i].bounds[1]; + params.stepscale[new_block_start + i] = new_params[i].stepscale; + params.flatprior[new_block_start + i] = new_params[i].flatprior; + params.isfree[new_block_start + i] = new_params[i].isfree; + params.covariance(new_block_start + i, new_block_start + i) = + new_params[i].error * new_params[i].error; + } + + params.inv_covariance = Eigen::MatrixXd(); +} + +void ParameterList::SetParameterCorrelation(int pidi, int pidj, double corr) { + if (pidi == pidj) { + MACH3LOG_ERROR( + "AddParameterCorrelation cannot be used to set covariance " + "matrix diagonal elements: ({0},{0}) attempted to be set to {1}", + pidi, corr); + throw MaCh3Exception(__FILE__, __LINE__); + } + + params.covariance(pidi, pidj) = + corr * + std::sqrt(params.covariance(pidi, pidi) * params.covariance(pidj, pidj)); + params.covariance(pidj, pidi) = params.covariance(pidi, pidj); + + params.inv_covariance = Eigen::MatrixXd(); +} + +void ParameterList::SetParameterAllCorrelations( + int paramid, std::map const &correlations) { + + params.covariance.row(paramid) = + Eigen::VectorXd::Zero(params.covariance.rows()); + params.covariance.col(paramid) = + Eigen::VectorXd::Zero(params.covariance.rows()); + + params.covariance(paramid, paramid) = + params.error[paramid] * params.error[paramid]; + + for (auto const &[other_name, corr] : correlations) { + SetParameterCorrelation(paramid, FindParameter(other_name), corr); + } +} + +int ParameterList::FindParameter(std::string const &name) { + auto pit = std::find(params.name.begin(), params.name.end(), name); + if (pit == params.name.end()) { + MACH3LOG_ERROR("ParameterList manages no parameter named {}.", name); + throw MaCh3Exception(__FILE__, __LINE__); + } + return int(std::distance(params.name.begin(), pit)); +} + +void ParameterList::ConstructTruncatedPCA(double threshold, int first, + int last) { + + M3::EnsureNoOutOfBlockCorrelations(params.covariance, {first, last}); + + pca.enabled = true; + pca.first_index = first; + pca.last_index = last; + pca.ntail = int(params.prefit.size() - last + 1); + + int blocksize = last - first + 1; + + pca.ortho_to_syst_rotation = CalculateTruncatedPCARotation( + params.covariance.block(pca.first_index, pca.first_index, blocksize, + blocksize), + threshold); +} + +void ParameterList::RotateOrthogonalParameterValuesToSystematicBasis( + Eigen::ArrayXd const &ortho_vals, Eigen::ArrayXd &systematic_vals) { + + systematic_vals.head(pca.first_index) = ortho_vals.head(pca.first_index); + + // 1) rotate the orthonogonal parameter values into 'deltas' in the + // systematic basis into the by using the PCA matrix. + // 2) add on the prefit values from to these deltas to return to the + // systematic basis. + systematic_vals.segment(pca.first_index, pca.nrotated_syst_parameters()) = + params.prefit.segment(pca.first_index, pca.nrotated_syst_parameters()) + + (ortho_vals.segment(pca.first_index, pca.northo_parameters()).matrix() * + pca.ortho_to_syst_rotation) + .array(); + + systematic_vals.tail(pca.ntail) = ortho_vals.tail(pca.ntail); +} + +void ParameterList::RotateSystematicParameterValuesToOrthogonalBasis( + Eigen::ArrayXd const &systematic_vals, Eigen::ArrayXd &ortho_vals) { + + ortho_vals.head(pca.first_index) = systematic_vals.head(pca.first_index); + + // 1) subtract the prefit values from the current parameter values so that the + // orthogonal basis has a CV of 0 + // 2) rotate the resulting parameter 'deltas' in the systematic basis into the + // orthogonal basis by using the transpose of the PCA matrix + ortho_vals.segment(pca.first_index, pca.northo_parameters()) = + ((systematic_vals.segment(pca.first_index, + pca.nrotated_syst_parameters()) - + params.prefit.segment(pca.first_index, pca.nrotated_syst_parameters())) + .matrix() * + pca.ortho_to_syst_rotation.transpose()) + .array(); + + ortho_vals.tail(pca.ntail) = systematic_vals.tail(pca.ntail); +} + +int ParameterList::SystematicParameterIndexToOrthogonalIndex(int i) { + if ((!pca.enabled) || (i < pca.first_index)) { + return i; + } else if ((i >= pca.first_index) && (i <= pca.last_index)) { + return ParameterInPCABlock; + } else { + return i - pca.nrotated_syst_parameters() + pca.northo_parameters(); + } +} + +double ParameterList::Chi2(Eigen::ArrayXd const &systematic_vals) { + + if (!params.inv_covariance.size()) { + params.inv_covariance = params.covariance.inverse(); + } + + auto diff = + params.flatprior.select(Eigen::VectorXd::Zero(params.prefit.size()), + systematic_vals - params.prefit); + return diff.transpose() * params.inv_covariance * diff; +} + +StepProposer ParameterList::MakeProposer() { + StepProposer proposer; + + if (!pca.enabled) { + proposer.SetProposalMatrix(params.covariance); + proposer.SetParameterValues(params.prefit); + } else { + // if we are using a PCA rotation then the proposer should only be aware of + // the orthogonal parameter basis. + + // copy the unrotated parameters and covariance blocks + int nproposer_parameters = + pca.first_index + pca.northo_parameters() + pca.ntail; + + Eigen::ArrayXd prefit_ortho = Eigen::ArrayXd::Zero(nproposer_parameters); + + RotateSystematicParameterValuesToOrthogonalBasis(params.prefit, + prefit_ortho); + + proposer.SetParameterValues(prefit_ortho); + + Eigen::MatrixXd covariance_ortho = + Eigen::MatrixXd::Zero(nproposer_parameters, nproposer_parameters); + + covariance_ortho.topLeftCorner(pca.first_index, pca.first_index) = + params.covariance.topLeftCorner(pca.first_index, pca.first_index); + + covariance_ortho.block(pca.first_index, pca.first_index, + pca.northo_parameters(), pca.northo_parameters()) = + Eigen::MatrixXd::Identity(pca.northo_parameters(), + pca.northo_parameters()); + + covariance_ortho.bottomRightCorner(pca.ntail, pca.ntail) = + params.covariance.bottomRightCorner(pca.ntail, pca.ntail); + + proposer.SetProposalMatrix(covariance_ortho); + } + + for (int i = 0; i < params.prefit.size(); ++i) { + + int idx = SystematicParameterIndexToOrthogonalIndex(i); + + if (params.flip_pivot[i].first) { + if (idx == ParameterInPCABlock) { + MACH3LOG_ERROR("Parameter, {}, index: {} is in the PCA block but " + "contained a special flip proposal definition", + params.name[i], i); + throw MaCh3Exception(__FILE__, __LINE__); + } + + proposer.special_proposal.flips.push_back( + {idx, params.flip_pivot[i].second}); + } + + if (std::get<0>(params.circ_bounds[i])) { + if (idx == ParameterInPCABlock) { + MACH3LOG_ERROR( + "Parameter, {}, index: {} is in the PCA block but " + "contained a special circular bounds proposal definition", + params.name[i], i); + throw MaCh3Exception(__FILE__, __LINE__); + } + proposer.special_proposal.circ_bounds.push_back( + {idx, std::get<1>(params.circ_bounds[i]), + std::get<2>(params.circ_bounds[i])}); + } + } + + return proposer; +} + +std::string ParameterList::SystematicParameterToString(int i) { + return fmt::format( + R"(name: {}, fancy_name: {} + prefit: {:.3g}, error: {:.3g}, bounds: [ {:.3g}, {:.3g} ], stepscale: {:.3g} + flatprior: {}, isfree: {} + is_pca: {}, has_flip: {}, has_circ_bounds: {})", + params.name[i], params.fancy_name[i], params.prefit[i], params.error[i], + params.lowbound[i], params.upbound[i], params.stepscale[i], + params.flatprior[i], params.isfree[i], + SystematicParameterIndexToOrthogonalIndex(i) == ParameterInPCABlock, + params.flip_pivot[i].first, std::get<0>(params.circ_bounds[i])); +} + +ParameterList MakeFromYAML(YAML::Node const &config) { + + ParameterList parlist; + + parlist.config.YAMLDoc = config; + + std::map> parameter_correlations; + std::vector param_infos; + + // ETA - read in the systematics. + for (auto const &node : parlist.config.YAMLDoc["Systematics"]) { + int param_id = int(param_infos.size()); + + auto const &pardef = node["Systematic"]; + + auto fancy_name = + Get(pardef["Names"]["FancyName"], __FILE__, __LINE__); + auto prefit = Get(pardef["ParameterValues"]["PreFitValue"], + __FILE__, __LINE__); + + auto error = Get(pardef["Error"], __FILE__, __LINE__); + + if (error <= 0) { + MACH3LOG_ERROR("Error for param {}({}) is negative and equal to {}", + fancy_name, param_id, error); + throw MaCh3Exception(__FILE__, __LINE__); + } + + auto stepscale = + Get(pardef["StepScale"]["MCMC"], __FILE__, __LINE__); + + auto bounds = GetBounds(pardef["ParameterBounds"]); + + auto flatprior = + GetFromManager(pardef["FlatPrior"], false, __FILE__, __LINE__); + + auto samplenames = GetFromManager>( + pardef["SampleNames"], {}, __FILE__, __LINE__); + + // Allow to fix param, this setting should be used only for params which are + // permanently fixed like baseline, please use global config for fixing + // param more flexibly + auto fixed = + GetFromManager(pardef["FixParam"], false, __FILE__, __LINE__); + + std::tuple param_circ_bounds{false, 0, 0}; + std::pair param_flip_pivot{false, 0}; + if (pardef["SpecialProposal"]) { + + bool CircEnabled = bool(pardef["SpecialProposal"]["CircularBounds"]); + bool FlipEnabled = bool(pardef["SpecialProposal"]["FlipParameter"]); + + if (!CircEnabled && !FlipEnabled) { + MACH3LOG_ERROR( + "None of Special Proposal were enabled even though param " + "{}, has SpecialProposal entry in Yaml", + fancy_name); + throw MaCh3Exception(__FILE__, __LINE__); + } + + if (CircEnabled) { + auto circular_bounds = Get>( + pardef["SpecialProposal"]["CircularBounds"], __FILE__, __LINE__); + // KS: Make sure circular circular_bounds are within physical + // circular_bounds. If we are outside of physics bound MCMC will never + // explore such phase space region + if ((circular_bounds[0] < bounds[0]) || + (circular_bounds[1] >= bounds[1])) { + MACH3LOG_ERROR( + "Circular circular_bounds [{}, {}] for parameter {} exceed " + "physical circular_bounds [{}, {}]", + circular_bounds[0], circular_bounds[1], fancy_name, bounds[0], + bounds[1]); + throw MaCh3Exception(__FILE__, __LINE__); + } + + param_circ_bounds = + std::make_tuple(true, circular_bounds[0], circular_bounds[1]); + + MACH3LOG_INFO( + "Enabling CircularBounds for parameter {} with range [{}, {}]", + fancy_name, circular_bounds[0], circular_bounds[1]); + } + + if (FlipEnabled) { + param_flip_pivot = std::make_pair( + true, Get(pardef["SpecialProposal"]["FlipParameter"], + __FILE__, __LINE__)); + + MACH3LOG_INFO("Enabling Flipping for parameter {} with value {}", + fancy_name, param_flip_pivot.second); + } + + if (CircEnabled && FlipEnabled) { + + const double fp = param_flip_pivot.second; + const double low = std::get<1>(param_circ_bounds); + const double high = std::get<2>(param_circ_bounds); + + if (fp < low || fp > high) { + MACH3LOG_ERROR( + "FlipParameter value {} for parameter {} is outside the " + "CircularBounds [{}, {}]", + fp, fancy_name, low, high); + throw MaCh3Exception(__FILE__, __LINE__); + } + + // Sanity check: ensure flipping any x in [low, high] keeps the result + // in [low, high] + const double flipped_low = 2 * fp - low; + const double flipped_high = 2 * fp - high; + const double min_flip = std::min(flipped_low, flipped_high); + const double max_flip = std::max(flipped_low, flipped_high); + + if (min_flip < low || max_flip > high) { + MACH3LOG_ERROR("Flipping about point {} for parameter {} would leave " + "circular bounds [{}, {}]", + param_flip_pivot.second, fancy_name, low, high); + throw MaCh3Exception(__FILE__, __LINE__); + } + } + } // end special proposal handling + + if (pardef["Correlations"]) { + for (auto const &corrn : pardef["Correlations"]) { + for (auto const &corr : corrn) { + parameter_correlations[param_id][corr.first.as()] = + corr.second.as(); + } + } + } + + param_infos.emplace_back(ParameterList::ParamInfo{fancy_name, + fancy_name, + prefit, + error, + stepscale, + {bounds[0], bounds[1]}, + flatprior, + !fixed, + samplenames, + param_flip_pivot, + param_circ_bounds}); + } + + parlist.AddParameters(param_infos); + + for (auto const &[paramid, correlations] : parameter_correlations) { + parlist.SetParameterAllCorrelations(paramid, correlations); + } + + parlist.params.covariance = M3::MakeMatrixPosDef(parlist.params.covariance); + + MACH3LOG_INFO("----------------"); + MACH3LOG_INFO("Found {} systematics parameters in total", + parlist.params.prefit.size()); + MACH3LOG_INFO("----------------"); + + return parlist; +} + +ParameterList MakeFromYAML(const std::vector &YAMLFiles) { + + MACH3LOG_INFO("Constructing instance of ParameterHandler using: "); + for (auto const &yf : YAMLFiles) { + MACH3LOG_INFO(" {}", yf); + } + MACH3LOG_INFO("as an input."); + + YAML::Node YAMLDoc; + YAMLDoc["Systematics"] = YAML::Node(YAML::NodeType::Sequence); + for (auto const &yfn : YAMLFiles) { + for (const auto &node : M3OpenConfig(yfn)["Systematics"]) { + YAMLDoc["Systematics"].push_back(node); + } + } + + auto parlist = MakeFromYAML(YAMLDoc); + parlist.config.inputFiles = YAMLFiles; + + MACH3LOG_INFO("Created parameter list from files: "); + for (const auto &file : parlist.config.inputFiles) { + MACH3LOG_INFO("{} ", file); + } + + return parlist; +} + +ParameterList MakeFromTMatrix(std::string const &name, + std::string const &file) { + + ParameterList parlist; + + // Set the covariance matrix from input ROOT file (e.g. flux, ND280, NIWG) + std::unique_ptr infile{TFile::Open(file.c_str(), "READ")}; + if (infile->IsZombie()) { + MACH3LOG_ERROR("Could not open input covariance ROOT file {} !!!", file); + MACH3LOG_ERROR("Was about to retrieve matrix with name {}", name); + throw MaCh3Exception(__FILE__, __LINE__); + } + + TMatrixDSym *CovMat = static_cast(infile->Get(name.c_str())); + + if (!CovMat) { + MACH3LOG_ERROR("Could not find covariance matrix name {} in file {}", name, + file); + MACH3LOG_ERROR("Are you really sure {} exists in the file?", name); + throw MaCh3Exception(__FILE__, __LINE__); + } + + std::vector param_infos; + for (int i = 0; i < CovMat->GetNrows(); ++i) { + param_infos.emplace_back(ParameterList::ParamInfo{ + std::string(CovMat->GetName()) + "_" + std::to_string(i), + "", + 0, + std::sqrt((*CovMat)(i, i)), + 1, + {-std::numeric_limits::max(), + std::numeric_limits::max()}, + false, + true, + {}, + {false, 0}, + {false, 0, 0}}); + } + parlist.AddParameters(param_infos); + + parlist.params.covariance = M3::MakeMatrixPosDef(M3::ROOTToEigen(*CovMat)); + + return parlist; +} diff --git a/Parameters/ParameterList.h b/Parameters/ParameterList.h new file mode 100644 index 000000000..9dc41f8d4 --- /dev/null +++ b/Parameters/ParameterList.h @@ -0,0 +1,105 @@ +#pragma once + +#include "Parameters/StepProposer.h" + +#include "Manager/Manager.h" + +#include "TMatrixDSym.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Walloca" +#pragma GCC diagnostic ignored "-Wold-style-cast" +#pragma GCC diagnostic ignored "-Wuseless-cast" +#pragma GCC diagnostic ignored "-Wconversion" +#pragma GCC diagnostic ignored "-Wduplicated-branches" +#pragma GCC diagnostic ignored "-Wstrict-aliasing" +#include "Eigen/Dense" +#pragma GCC diagnostic pop + +#include +#include +#include +#include +#include + +struct ParameterList { + + struct { + std::vector name, fancy_name; + Eigen::ArrayXd prefit, error, lowbound, upbound, stepscale; + Eigen::ArrayXi flatprior, isfree; + std::vector> samples; + + Eigen::MatrixXd covariance; + Eigen::MatrixXd inv_covariance; + + // enabled, flip pivot value + std::vector> flip_pivot; + // enabled, low bound, up bound + std::vector> circ_bounds; + } params; + + ParameterList() {}; + + struct ParamInfo { + std::string name, fancy_name; + double prefit, error, stepscale; + std::array bounds; + bool flatprior, isfree; + std::vector affected_samples; + std::pair flip_pivot; + std::tuple circ_bounds; + }; + + void AddParameters(std::vector const &new_params); + void AddParameter(ParamInfo const &new_param) { + AddParameters({ + new_param, + }); + } + + std::string SystematicParameterToString(int i); + + void SetParameterCorrelation(int pidi, int pidj, double corr); + void SetParameterAllCorrelations( + int paramid, std::map const &correlations); + + int FindParameter(std::string const &name); + + struct { + bool enabled; + int first_index, last_index, ntail; + Eigen::MatrixXd ortho_to_syst_rotation; + + int nrotated_syst_parameters() const { + return int(ortho_to_syst_rotation.cols()); + } + int northo_parameters() const { return int(ortho_to_syst_rotation.rows()); } + } pca; + + void ConstructTruncatedPCA(double threshold, int first, int last); + + void RotateOrthogonalParameterValuesToSystematicBasis( + Eigen::ArrayXd const &ortho_vals, Eigen::ArrayXd &systematic_vals); + + void RotateSystematicParameterValuesToOrthogonalBasis( + Eigen::ArrayXd const &systematic_vals, Eigen::ArrayXd &ortho_vals); + + constexpr static int ParameterInPCABlock = std::numeric_limits::max(); + int SystematicParameterIndexToOrthogonalIndex(int i); + + double Chi2(Eigen::ArrayXd const &systematic_vals); + + StepProposer MakeProposer(); + + struct { + /// The input root file we read in + std::vector inputFiles; + /// Stores config describing systematics + YAML::Node YAMLDoc; + } config; +}; + +ParameterList MakeFromYAML(YAML::Node const &config); +ParameterList MakeFromYAML(const std::vector &YAMLFiles); +ParameterList MakeFromTMatrix(std::string const &name, std::string const &file); diff --git a/Parameters/README.md b/Parameters/README.md new file mode 100644 index 000000000..f6fa5c330 --- /dev/null +++ b/Parameters/README.md @@ -0,0 +1,14 @@ +# Parameters + +## ParameterHandlerBase Design Decisions + +Elementary objects needed for sampling: +- Parameter list + - Read from YAML. Knows everything about each systematic parameter: name, error, possibly-correlated prior, restrictions on parameter applicability + +- StepProposer + - Knows nothing about the meaning of parameters + - Generates parameter values that are steps away from some current point according to a proposal matrix. + - Keeps track of parameter step scales and possibly updates the proposal matrix as steps are accepted. + - Possibly steps in a rotated parameter space. + - Possibly has 'special proposals': flips and circular diff --git a/Parameters/StepProposer.cpp b/Parameters/StepProposer.cpp new file mode 100644 index 000000000..790327d00 --- /dev/null +++ b/Parameters/StepProposer.cpp @@ -0,0 +1,66 @@ +#include "Parameters/StepProposer.h" + +StepProposer::StepProposer() { + special_proposal.enabled = false; + adaptive.enabled = false; + + rng.gaus = std::normal_distribution(0); + rng.unif = std::uniform_real_distribution(0, 1); +}; + +StepProposer::StepProposer(Eigen::MatrixXd proposal_matrix, + Eigen::ArrayXd current_values) + : StepProposer() { + SetProposalMatrix(proposal_matrix); + SetParameterValues(current_values); +} + +Eigen::ArrayXd const &StepProposer::Propose() { + + Eigen::VectorXd random_vector = + Eigen::VectorXd::NullaryExpr(params.current.size(), [&](int i) { + return params.isfree[i] ? rng.gaus(rng.e1) : 0; + }); + + params.proposed = + params.current + ((params.l_proposal * random_vector).array() * + params.scale * params.global_scale); + + // if we don't have any special proposal functions, leave early + if (!special_proposal.enabled) { + return params.proposed; + } + + for (auto const &[i, low, up] : special_proposal.circ_bounds) { + if (params.isfree[i]) { + continue; + } + if (params.proposed[i] > up) { + params.proposed[i] = low + std::fmod(params.proposed[i] - up, up - low); + } else if (params.proposed[i] < low) { + params.proposed[i] = up - std::fmod(low - params.proposed[i], up - low); + } + } + + for (auto const &[i, pivot] : special_proposal.flips) { + if (params.isfree[i]) { + continue; + } + if (rng.unif(rng.e1) < 0.5) { + params.proposed[i] = 2 * pivot - params.proposed[i]; + } + } + + return params.proposed; +} + +void StepProposer::SetProposalMatrix(Eigen::MatrixXd proposal_matrix) { + Eigen::LLT lltproposal(proposal_matrix); + if (lltproposal.info() != Eigen::Success) { + MACH3LOG_ERROR("Failed to LLT decompose proposal matrix"); + throw MaCh3Exception(__FILE__, __LINE__); + } + params.l_proposal = lltproposal.matrixL(); +} + +void StepProposer::Accept() {} diff --git a/Parameters/StepProposer.h b/Parameters/StepProposer.h new file mode 100644 index 000000000..21b89081a --- /dev/null +++ b/Parameters/StepProposer.h @@ -0,0 +1,70 @@ +#pragma once + +#include "Manager/Manager.h" + +#include "TMatrixDSym.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Walloca" +#pragma GCC diagnostic ignored "-Wold-style-cast" +#pragma GCC diagnostic ignored "-Wuseless-cast" +#pragma GCC diagnostic ignored "-Wconversion" +#include "Eigen/Dense" +#pragma GCC diagnostic pop + +#include +#include +#include + +struct StepProposer { + struct { + Eigen::ArrayXd current, proposed, scale; + Eigen::ArrayXi isfree; + double global_scale; + Eigen::MatrixXd l_proposal; + } params; + + struct { + bool enabled; + + // param_index, flip pivot value + std::vector> flips; + // param_index, low bound, up bound + std::vector> circ_bounds; + + } special_proposal; + + struct { + std::ranlux48 e1; + std::normal_distribution gaus; + std::uniform_real_distribution unif; + } rng; + + struct { + TMatrixDSym proposal; + } root_copies; + + struct { + bool enabled; + size_t nsteps; + Eigen::MatrixXd accepted_parameter_covariance; + } adaptive; + + StepProposer(); + StepProposer(Eigen::MatrixXd proposal_matrix, Eigen::ArrayXd current_values); + + void SetProposalMatrix(Eigen::MatrixXd proposal_matrix); + void SetParameterValues(Eigen::ArrayXd current_values) { + params.current = current_values; + } + + void EnableAdaption(YAML::Node const &adaption_config); + void SetAdaptionCovariance(Eigen::MatrixXd parameter_covariance, + size_t nsteps) { + adaptive.nsteps = nsteps; + adaptive.accepted_parameter_covariance = parameter_covariance; + } + + Eigen::ArrayXd const &Propose(); + void Accept(); +}; diff --git a/Parameters/test/CMakeLists.txt b/Parameters/test/CMakeLists.txt new file mode 100644 index 000000000..344741244 --- /dev/null +++ b/Parameters/test/CMakeLists.txt @@ -0,0 +1,19 @@ +CPMAddPackage( + NAME Catch2 + GITHUB_REPOSITORY catchorg/Catch2 + VERSION 3.3.2 +) + +LIST(APPEND CMAKE_MODULE_PATH ${Catch2_SOURCE_DIR}/extras) + +include(CTest) +include(Catch) + +add_executable(ParameterListTest ParameterListTest.cpp) +target_link_libraries(ParameterListTest PRIVATE Catch2::Catch2WithMain Parameters) +catch_discover_tests(ParameterListTest) + + +# add_executable(StepProposerTest StepProposerTest.cxx) +# target_link_libraries(StepProposerTest PRIVATE Catch2::Catch2WithMain Parameters) +# catch_discover_tests(StepProposerTest) diff --git a/Parameters/test/ParameterListTest.cpp b/Parameters/test/ParameterListTest.cpp new file mode 100644 index 000000000..23cded8af --- /dev/null +++ b/Parameters/test/ParameterListTest.cpp @@ -0,0 +1,46 @@ +#include "Parameters/ParameterList.h" + +#include "catch2/catch_test_macros.hpp" +#include "catch2/matchers/catch_matchers_floating_point.hpp" + +#include + +using namespace Catch::Matchers; + +YAML::Node BuildConfig() { + return YAML::Load( + R"( +Systematics: +- Systematic: { + Names: { FancyName: par1 }, + ParameterValues: { PreFitValue: 1 }, + Error: 0.5, + StepScale: { MCMC: 0.1 }, + ParameterBounds: [ -3, 3 ], + FlatPrior: False, + SampleNames: [ "A", "B" ], + FixParam: False, + SpecialProposal: { + CircularBounds: [ -2.5, 2.5 ], + FlipParameter: 0 + }, + Correlations: [ { par2: 0.5 }, ] + } +- Systematic: { + Names: { FancyName: par2 }, + ParameterValues: { PreFitValue: 1 }, + Error: 0.5, + StepScale: { MCMC: 0.1 }, + ParameterBounds: [ -2, 2 ], + Correlations: [ { par1: 0.5 }, ] + } +)"); +} + +TEST_CASE("MakeFromYAML", "[Construction]") { + auto parlist = MakeFromYAML(BuildConfig()); + + for (int i = 0; i < parlist.params.prefit.size(); ++i) { + std::cout << parlist.SystematicParameterToString(i) << std::endl; + } +} diff --git a/Parameters/test/StepProposerTest.cpp b/Parameters/test/StepProposerTest.cpp new file mode 100644 index 000000000..fcab4939f --- /dev/null +++ b/Parameters/test/StepProposerTest.cpp @@ -0,0 +1 @@ +StepProposerTest.cpp From 0e02f8e220ee9995bd0f4f34cf9fd07c16c6ffce Mon Sep 17 00:00:00 2001 From: Luke Pickering Date: Tue, 16 Dec 2025 17:10:35 +0000 Subject: [PATCH 03/14] Add some useful interface functions for parameter numbers --- Parameters/ParameterList.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Parameters/ParameterList.h b/Parameters/ParameterList.h index 9dc41f8d4..b57f203d2 100644 --- a/Parameters/ParameterList.h +++ b/Parameters/ParameterList.h @@ -65,6 +65,7 @@ struct ParameterList { int paramid, std::map const &correlations); int FindParameter(std::string const &name); + int NumSystematicBasisParameters() { return params.prefit.size(); } struct { bool enabled; @@ -88,6 +89,13 @@ struct ParameterList { constexpr static int ParameterInPCABlock = std::numeric_limits::max(); int SystematicParameterIndexToOrthogonalIndex(int i); + int NumOrthogonalBasisParameters() { + return pca.enabled + ? (NumSystematicBasisParameters() - + pca.nrotated_syst_parameters() + pca.northo_parameters()) + : NumSystematicBasisParameters(); + } + double Chi2(Eigen::ArrayXd const &systematic_vals); StepProposer MakeProposer(); From 863f1a678e232436a4c998d2647717875e257141 Mon Sep 17 00:00:00 2001 From: Luke Pickering Date: Wed, 17 Dec 2025 07:44:00 +0000 Subject: [PATCH 04/14] Adds a number of test cases and fixes a few small issues they exposed. --- Parameters/PCA.h | 11 +- Parameters/ParameterList.cpp | 101 +++++++----- Parameters/ParameterList.h | 22 +-- Parameters/test/ParameterListTest.cpp | 225 ++++++++++++++++++++++++-- 4 files changed, 298 insertions(+), 61 deletions(-) diff --git a/Parameters/PCA.h b/Parameters/PCA.h index c23a1ee6a..134f3e404 100644 --- a/Parameters/PCA.h +++ b/Parameters/PCA.h @@ -13,7 +13,7 @@ #include #include -Eigen::MatrixXd CalculateTruncatedPCARotation(Eigen::MatrixXd mx_phyiscs_basis, +inline Eigen::MatrixXd CalculateTruncatedPCARotation(Eigen::MatrixXd mx_phyiscs_basis, double threshold) { // a covariance is real symmetric, so self adjoint @@ -31,10 +31,13 @@ Eigen::MatrixXd CalculateTruncatedPCARotation(Eigen::MatrixXd mx_phyiscs_basis, evals.emplace_back(i, eigen_val[i]); } + std::cout << "PCA:\n evals: " << eigen_val << "\n evects:\n" + << eigen_vect << std::endl; + // sorting is not strictly neccessary but orders the orthogonal parameter // basis approximately by uncertainty. std::sort( - evals.begin(), evals.end(), + evals.rbegin(), evals.rend(), [](std::pair const &l, std::pair const &r) { return l.second < r.second; }); @@ -42,8 +45,10 @@ Eigen::MatrixXd CalculateTruncatedPCARotation(Eigen::MatrixXd mx_phyiscs_basis, double evalsum = 0; int northo_parameters = 0; + double trace = mx_phyiscs_basis.trace(); + for (auto &[i, ev] : evals) { - if (std::fabs(ev) > threshold) { + if (std::fabs(ev / trace) > threshold) { evalsum += std::fabs(ev); northo_parameters++; } else { diff --git a/Parameters/ParameterList.cpp b/Parameters/ParameterList.cpp index 8512ff879..c6863be03 100644 --- a/Parameters/ParameterList.cpp +++ b/Parameters/ParameterList.cpp @@ -90,61 +90,66 @@ void ParameterList::ConstructTruncatedPCA(double threshold, int first, pca.enabled = true; pca.first_index = first; pca.last_index = last; - pca.ntail = int(params.prefit.size() - last + 1); + pca.ntail = int(params.prefit.size() - (last + 1)); - int blocksize = last - first + 1; + int blocksize = (last + 1) - first; - pca.ortho_to_syst_rotation = CalculateTruncatedPCARotation( + pca.pc_to_syst_rotation = CalculateTruncatedPCARotation( params.covariance.block(pca.first_index, pca.first_index, blocksize, blocksize), threshold); + + MACH3LOG_INFO("ParameterList::ConstructTruncatedPCA: threshold = {}, " + "nsystparams = {} truncated to {} PC params.", + threshold, pca.nrotated_syst_parameters(), + pca.npc_parameters()); } -void ParameterList::RotateOrthogonalParameterValuesToSystematicBasis( - Eigen::ArrayXd const &ortho_vals, Eigen::ArrayXd &systematic_vals) { +void ParameterList::RotatePCParameterValuesToSystematicBasis( + Eigen::ArrayXd const &pc_vals, Eigen::ArrayXd &systematic_vals) { - systematic_vals.head(pca.first_index) = ortho_vals.head(pca.first_index); + systematic_vals.head(pca.first_index) = pc_vals.head(pca.first_index); - // 1) rotate the orthonogonal parameter values into 'deltas' in the + // 1) rotate the principle component parameter values into 'deltas' in the // systematic basis into the by using the PCA matrix. // 2) add on the prefit values from to these deltas to return to the // systematic basis. systematic_vals.segment(pca.first_index, pca.nrotated_syst_parameters()) = params.prefit.segment(pca.first_index, pca.nrotated_syst_parameters()) + - (ortho_vals.segment(pca.first_index, pca.northo_parameters()).matrix() * - pca.ortho_to_syst_rotation) + (pc_vals.segment(pca.first_index, pca.npc_parameters()).matrix() * + pca.pc_to_syst_rotation) .array(); - systematic_vals.tail(pca.ntail) = ortho_vals.tail(pca.ntail); + systematic_vals.tail(pca.ntail) = pc_vals.tail(pca.ntail); } -void ParameterList::RotateSystematicParameterValuesToOrthogonalBasis( - Eigen::ArrayXd const &systematic_vals, Eigen::ArrayXd &ortho_vals) { +void ParameterList::RotateSystematicParameterValuesToPCBasis( + Eigen::ArrayXd const &systematic_vals, Eigen::ArrayXd &pc_vals) { - ortho_vals.head(pca.first_index) = systematic_vals.head(pca.first_index); + pc_vals.head(pca.first_index) = systematic_vals.head(pca.first_index); // 1) subtract the prefit values from the current parameter values so that the - // orthogonal basis has a CV of 0 + // PC basis has a CV of 0 // 2) rotate the resulting parameter 'deltas' in the systematic basis into the - // orthogonal basis by using the transpose of the PCA matrix - ortho_vals.segment(pca.first_index, pca.northo_parameters()) = + // PC basis by using the transpose of the PCA matrix + pc_vals.segment(pca.first_index, pca.npc_parameters()) = ((systematic_vals.segment(pca.first_index, pca.nrotated_syst_parameters()) - params.prefit.segment(pca.first_index, pca.nrotated_syst_parameters())) .matrix() * - pca.ortho_to_syst_rotation.transpose()) + pca.pc_to_syst_rotation.transpose()) .array(); - ortho_vals.tail(pca.ntail) = systematic_vals.tail(pca.ntail); + pc_vals.tail(pca.ntail) = systematic_vals.tail(pca.ntail); } -int ParameterList::SystematicParameterIndexToOrthogonalIndex(int i) { +int ParameterList::SystematicParameterIndexToPCIndex(int i) { if ((!pca.enabled) || (i < pca.first_index)) { return i; } else if ((i >= pca.first_index) && (i <= pca.last_index)) { return ParameterInPCABlock; } else { - return i - pca.nrotated_syst_parameters() + pca.northo_parameters(); + return i - pca.nrotated_syst_parameters() + pca.npc_parameters(); } } @@ -168,39 +173,37 @@ StepProposer ParameterList::MakeProposer() { proposer.SetParameterValues(params.prefit); } else { // if we are using a PCA rotation then the proposer should only be aware of - // the orthogonal parameter basis. + // the PC parameter basis. // copy the unrotated parameters and covariance blocks int nproposer_parameters = - pca.first_index + pca.northo_parameters() + pca.ntail; + pca.first_index + pca.npc_parameters() + pca.ntail; - Eigen::ArrayXd prefit_ortho = Eigen::ArrayXd::Zero(nproposer_parameters); + Eigen::ArrayXd prefit_pc = Eigen::ArrayXd::Zero(nproposer_parameters); - RotateSystematicParameterValuesToOrthogonalBasis(params.prefit, - prefit_ortho); + RotateSystematicParameterValuesToPCBasis(params.prefit, prefit_pc); - proposer.SetParameterValues(prefit_ortho); + proposer.SetParameterValues(prefit_pc); - Eigen::MatrixXd covariance_ortho = + Eigen::MatrixXd covariance_pc = Eigen::MatrixXd::Zero(nproposer_parameters, nproposer_parameters); - covariance_ortho.topLeftCorner(pca.first_index, pca.first_index) = + covariance_pc.topLeftCorner(pca.first_index, pca.first_index) = params.covariance.topLeftCorner(pca.first_index, pca.first_index); - covariance_ortho.block(pca.first_index, pca.first_index, - pca.northo_parameters(), pca.northo_parameters()) = - Eigen::MatrixXd::Identity(pca.northo_parameters(), - pca.northo_parameters()); + covariance_pc.block(pca.first_index, pca.first_index, pca.npc_parameters(), + pca.npc_parameters()) = + Eigen::MatrixXd::Identity(pca.npc_parameters(), pca.npc_parameters()); - covariance_ortho.bottomRightCorner(pca.ntail, pca.ntail) = + covariance_pc.bottomRightCorner(pca.ntail, pca.ntail) = params.covariance.bottomRightCorner(pca.ntail, pca.ntail); - proposer.SetProposalMatrix(covariance_ortho); + proposer.SetProposalMatrix(covariance_pc); } for (int i = 0; i < params.prefit.size(); ++i) { - int idx = SystematicParameterIndexToOrthogonalIndex(i); + int idx = SystematicParameterIndexToPCIndex(i); if (params.flip_pivot[i].first) { if (idx == ParameterInPCABlock) { @@ -240,7 +243,7 @@ std::string ParameterList::SystematicParameterToString(int i) { params.name[i], params.fancy_name[i], params.prefit[i], params.error[i], params.lowbound[i], params.upbound[i], params.stepscale[i], params.flatprior[i], params.isfree[i], - SystematicParameterIndexToOrthogonalIndex(i) == ParameterInPCABlock, + SystematicParameterIndexToPCIndex(i) == ParameterInPCABlock, params.flip_pivot[i].first, std::get<0>(params.circ_bounds[i])); } @@ -392,6 +395,32 @@ ParameterList MakeFromYAML(YAML::Node const &config) { parlist.AddParameters(param_infos); for (auto const &[paramid, correlations] : parameter_correlations) { + + // check that we don't have incompatible correlations defined the other way + for (auto const &[oparam, corr] : correlations) { + int oparamid = parlist.FindParameter(oparam); + auto const &pname = parlist.params.name[paramid]; + auto const &opname = parlist.params.name[oparamid]; + if (!parameter_correlations.count(oparamid) || + !parameter_correlations.at(oparamid).count(pname)) { + MACH3LOG_ERROR("Encountered inconsistent correlations defined between " + "parameters {0} and {1}. {0} reports a correlation of " + "{2} with {1} that is not reciprocated.", + pname, opname, corr); + throw MaCh3Exception(__FILE__, __LINE__); + } + if (parameter_correlations.count(oparamid) && + parameter_correlations.at(oparamid).count(pname) && + std::fabs(parameter_correlations.at(oparamid).at(pname) - corr) > + 1E-8) { + MACH3LOG_ERROR("Encountered inconsistent correlations defined between " + "parameters {} and {}: {} and {}.", + pname, opname, corr, + parameter_correlations.at(oparamid).at(pname)); + throw MaCh3Exception(__FILE__, __LINE__); + } + } + parlist.SetParameterAllCorrelations(paramid, correlations); } diff --git a/Parameters/ParameterList.h b/Parameters/ParameterList.h index b57f203d2..c9f15a462 100644 --- a/Parameters/ParameterList.h +++ b/Parameters/ParameterList.h @@ -65,34 +65,34 @@ struct ParameterList { int paramid, std::map const &correlations); int FindParameter(std::string const &name); - int NumSystematicBasisParameters() { return params.prefit.size(); } + int NumSystematicBasisParameters() { return int(params.prefit.size()); } struct { bool enabled; int first_index, last_index, ntail; - Eigen::MatrixXd ortho_to_syst_rotation; + Eigen::MatrixXd pc_to_syst_rotation; int nrotated_syst_parameters() const { - return int(ortho_to_syst_rotation.cols()); + return int(pc_to_syst_rotation.cols()); } - int northo_parameters() const { return int(ortho_to_syst_rotation.rows()); } + int npc_parameters() const { return int(pc_to_syst_rotation.rows()); } } pca; void ConstructTruncatedPCA(double threshold, int first, int last); - void RotateOrthogonalParameterValuesToSystematicBasis( - Eigen::ArrayXd const &ortho_vals, Eigen::ArrayXd &systematic_vals); + void RotatePCParameterValuesToSystematicBasis( + Eigen::ArrayXd const &pc_vals, Eigen::ArrayXd &systematic_vals); - void RotateSystematicParameterValuesToOrthogonalBasis( - Eigen::ArrayXd const &systematic_vals, Eigen::ArrayXd &ortho_vals); + void RotateSystematicParameterValuesToPCBasis( + Eigen::ArrayXd const &systematic_vals, Eigen::ArrayXd &pc_vals); constexpr static int ParameterInPCABlock = std::numeric_limits::max(); - int SystematicParameterIndexToOrthogonalIndex(int i); + int SystematicParameterIndexToPCIndex(int i); - int NumOrthogonalBasisParameters() { + int NumPCBasisParameters() { return pca.enabled ? (NumSystematicBasisParameters() - - pca.nrotated_syst_parameters() + pca.northo_parameters()) + pca.nrotated_syst_parameters() + pca.npc_parameters()) : NumSystematicBasisParameters(); } diff --git a/Parameters/test/ParameterListTest.cpp b/Parameters/test/ParameterListTest.cpp index 23cded8af..052db1da3 100644 --- a/Parameters/test/ParameterListTest.cpp +++ b/Parameters/test/ParameterListTest.cpp @@ -7,14 +7,14 @@ using namespace Catch::Matchers; -YAML::Node BuildConfig() { - return YAML::Load( +TEST_CASE("MakeFromYAML", "[Construction]") { + auto parlist = MakeFromYAML(YAML::Load( R"( Systematics: - Systematic: { Names: { FancyName: par1 }, ParameterValues: { PreFitValue: 1 }, - Error: 0.5, + Error: 1, StepScale: { MCMC: 0.1 }, ParameterBounds: [ -3, 3 ], FlatPrior: False, @@ -24,23 +24,226 @@ YAML::Node BuildConfig() { CircularBounds: [ -2.5, 2.5 ], FlipParameter: 0 }, - Correlations: [ { par2: 0.5 }, ] + Correlations: [ { par2: 0.49 }, ] } - Systematic: { Names: { FancyName: par2 }, ParameterValues: { PreFitValue: 1 }, - Error: 0.5, + Error: 1, StepScale: { MCMC: 0.1 }, ParameterBounds: [ -2, 2 ], - Correlations: [ { par1: 0.5 }, ] + FlatPrior: True, + Correlations: [ { par1: 0.49 }, ] } -)"); +- Systematic: { + Names: { FancyName: par3 }, + ParameterValues: { PreFitValue: 0.25 }, + Error: 0.5, + StepScale: { MCMC: 1 }, + ParameterBounds: [ -2, 2 ], + } +)")); + + REQUIRE(parlist.FindParameter("par1") == 0); + REQUIRE(parlist.FindParameter("par2") == 1); + REQUIRE(parlist.FindParameter("par3") == 2); + + REQUIRE_THROWS_AS(parlist.FindParameter("par4"), MaCh3Exception); + + REQUIRE(parlist.params.name[0] == "par1"); + REQUIRE(parlist.params.lowbound[0] == -3); + REQUIRE(parlist.params.upbound[0] == 3); + REQUIRE(parlist.params.flip_pivot[0].first == true); + REQUIRE(parlist.params.flip_pivot[0].second == 0); + REQUIRE(std::get<0>(parlist.params.circ_bounds[0]) == true); + REQUIRE(std::get<1>(parlist.params.circ_bounds[0]) == -2.5); + REQUIRE(std::get<2>(parlist.params.circ_bounds[0]) == 2.5); + + REQUIRE(parlist.params.name[1] == "par2"); + REQUIRE(parlist.params.error[1] == 1); + REQUIRE(parlist.params.flatprior[1] == 1); + + REQUIRE(parlist.params.name[2] == "par3"); + REQUIRE_THAT(parlist.params.prefit[2], WithinAbs(0.25, 1E-8)); + REQUIRE_THAT(parlist.params.error[2], WithinAbs(0.5, 1E-8)); + REQUIRE_THAT(parlist.params.stepscale[2], WithinAbs(1, 1E-8)); + + REQUIRE_THAT(parlist.params.covariance(0, 0), WithinAbs(1, 1E-8)); + REQUIRE_THAT(parlist.params.covariance(1, 1), WithinAbs(1, 1E-8)); + REQUIRE_THAT(parlist.params.covariance(2, 2), WithinAbs(0.25, 1E-8)); + + REQUIRE_THAT(parlist.params.covariance(1, 0), WithinAbs(0.49, 1E-8)); + REQUIRE_THAT(parlist.params.covariance(0, 1), WithinAbs(0.49, 1E-8)); + REQUIRE_THAT(parlist.params.covariance(0, 2), WithinAbs(0, 1E-8)); + REQUIRE_THAT(parlist.params.covariance(1, 2), WithinAbs(0, 1E-8)); } -TEST_CASE("MakeFromYAML", "[Construction]") { - auto parlist = MakeFromYAML(BuildConfig()); +TEST_CASE("MakeFromYAML Bad Correlations", "[Construction]") { + REQUIRE_THROWS_AS(MakeFromYAML(YAML::Load( + R"( +Systematics: +- Systematic: { + Names: { FancyName: par1 }, + ParameterValues: { PreFitValue: 1 }, + Error: 1, + StepScale: { MCMC: 0.1 }, + ParameterBounds: [ -3, 3 ], + Correlations: [ { par2: 0.49 }, ] + } +- Systematic: { + Names: { FancyName: par2 }, + ParameterValues: { PreFitValue: 1 }, + Error: 1, + StepScale: { MCMC: 0.1 }, + ParameterBounds: [ -3, 3 ], + Correlations: [ { par1: 0.51 }, ] + } +)")), + MaCh3Exception); - for (int i = 0; i < parlist.params.prefit.size(); ++i) { - std::cout << parlist.SystematicParameterToString(i) << std::endl; + REQUIRE_THROWS_AS(MakeFromYAML(YAML::Load( + R"( +Systematics: +- Systematic: { + Names: { FancyName: par1 }, + ParameterValues: { PreFitValue: 1 }, + Error: 1, + StepScale: { MCMC: 0.1 }, + ParameterBounds: [ -3, 3 ], + Correlations: [ { par2: 0.49 }, ] + } +- Systematic: { + Names: { FancyName: par2 }, + ParameterValues: { PreFitValue: 1 }, + Error: 1, + StepScale: { MCMC: 0.1 }, + ParameterBounds: [ -3, 3 ], + Correlations: [ { par3: 0.51 }, ] + } +)")), + MaCh3Exception); + + REQUIRE_THROWS_AS(MakeFromYAML(YAML::Load( + R"( +Systematics: +- Systematic: { + Names: { FancyName: par1 }, + ParameterValues: { PreFitValue: 1 }, + Error: 1, + StepScale: { MCMC: 1 }, + ParameterBounds: [ -10, 10 ], + } +- Systematic: { + Names: { FancyName: par2 }, + ParameterValues: { PreFitValue: 2 }, + Error: 1, + StepScale: { MCMC: 1 }, + ParameterBounds: [ -10, 10 ], + Correlations: [ { par3: 0.9999 }, ] + } +- Systematic: { + Names: { FancyName: par3 }, + ParameterValues: { PreFitValue: 3 }, + Error: 1, + StepScale: { MCMC: 1 }, + ParameterBounds: [ -10, 10 ], + Correlations: [ { par2: 0.9999 }, + { par1: 0.1 }, ] + } +- Systematic: { + Names: { FancyName: par4 }, + ParameterValues: { PreFitValue: 4 }, + Error: 1, + StepScale: { MCMC: 1 }, + ParameterBounds: [ -10, 10 ], + } +)")), + MaCh3Exception); +} + +TEST_CASE("4param", "[PCA]") { + auto parlist = MakeFromYAML(YAML::Load( + R"( +Systematics: +- Systematic: { + Names: { FancyName: par1 }, + ParameterValues: { PreFitValue: 1 }, + Error: 1, + StepScale: { MCMC: 1 }, + ParameterBounds: [ -10, 10 ], + } +- Systematic: { + Names: { FancyName: par2 }, + ParameterValues: { PreFitValue: 2 }, + Error: 1, + StepScale: { MCMC: 1 }, + ParameterBounds: [ -10, 10 ], + Correlations: [ { par3: 0.9999 }, ] + } +- Systematic: { + Names: { FancyName: par3 }, + ParameterValues: { PreFitValue: 3 }, + Error: 1, + StepScale: { MCMC: 1 }, + ParameterBounds: [ -10, 10 ], + Correlations: [ { par2: 0.9999 }, ] + } +- Systematic: { + Names: { FancyName: par4 }, + ParameterValues: { PreFitValue: 4 }, + Error: 1, + StepScale: { MCMC: 1 }, + ParameterBounds: [ -10, 10 ], + } +)")); + + parlist.ConstructTruncatedPCA(1E-4, 1, 2); + + REQUIRE(parlist.pca.enabled); + REQUIRE(parlist.pca.first_index == 1); + REQUIRE(parlist.pca.last_index == 2); + REQUIRE(parlist.pca.ntail == 1); + REQUIRE(parlist.pca.nrotated_syst_parameters() == 2); + REQUIRE(parlist.pca.npc_parameters() == 1); + REQUIRE(parlist.NumPCBasisParameters() == 3); +} + +TEST_CASE("Out of block correlations", "[PCA]") { + auto parlist = MakeFromYAML(YAML::Load( + R"( +Systematics: +- Systematic: { + Names: { FancyName: par1 }, + ParameterValues: { PreFitValue: 1 }, + Error: 1, + StepScale: { MCMC: 1 }, + ParameterBounds: [ -10, 10 ], + Correlations: [ { par3: 0.3 }, ] + } +- Systematic: { + Names: { FancyName: par2 }, + ParameterValues: { PreFitValue: 2 }, + Error: 1, + StepScale: { MCMC: 1 }, + ParameterBounds: [ -10, 10 ], + Correlations: [ { par3: 0.5 }, ] + } +- Systematic: { + Names: { FancyName: par3 }, + ParameterValues: { PreFitValue: 3 }, + Error: 1, + StepScale: { MCMC: 1 }, + ParameterBounds: [ -10, 10 ], + Correlations: [ { par2: 0.5 }, + { par1: 0.3 }, ] + } +- Systematic: { + Names: { FancyName: par4 }, + ParameterValues: { PreFitValue: 4 }, + Error: 1, + StepScale: { MCMC: 1 }, + ParameterBounds: [ -10, 10 ], } +)")); + REQUIRE_THROWS_AS(parlist.ConstructTruncatedPCA(1E-4, 1, 2), MaCh3Exception); } From 8aea8f29c305ecc1b3bb66296a5e459696167189 Mon Sep 17 00:00:00 2001 From: Luke Pickering Date: Wed, 17 Dec 2025 21:34:20 +0000 Subject: [PATCH 05/14] Implements most of ParameterHandlerBase interface in terms of new classes - Some particularly egregious interface points throw runtime errors. Can implement if we really really need to. - More test cases, including an RNG test of the step proposer and PCA rotation back and forth. - some const qualifying... --- Parameters/CMakeLists.txt | 2 +- Parameters/PCA.h | 55 +- Parameters/ParameterHandlerBase.cpp | 1040 +++++++++---------------- Parameters/ParameterHandlerBase.h | 326 +++----- Parameters/ParameterList.cpp | 89 ++- Parameters/ParameterList.h | 37 +- Parameters/StepProposer.cpp | 5 +- Parameters/StepProposer.h | 21 +- Parameters/test/CMakeLists.txt | 7 +- Parameters/test/ParameterListTest.cpp | 55 +- Parameters/test/StepProposerTest.cpp | 120 ++- 11 files changed, 757 insertions(+), 1000 deletions(-) diff --git a/Parameters/CMakeLists.txt b/Parameters/CMakeLists.txt index 5484e97ad..e15d1037f 100644 --- a/Parameters/CMakeLists.txt +++ b/Parameters/CMakeLists.txt @@ -9,7 +9,7 @@ set(HEADERS ) add_library(Parameters SHARED - # ParameterHandlerBase.cpp + ParameterHandlerBase.cpp # ParameterHandlerGeneric.cpp ParameterTunes.cpp StepProposer.cpp diff --git a/Parameters/PCA.h b/Parameters/PCA.h index 134f3e404..3dc66feb3 100644 --- a/Parameters/PCA.h +++ b/Parameters/PCA.h @@ -13,11 +13,13 @@ #include #include -inline Eigen::MatrixXd CalculateTruncatedPCARotation(Eigen::MatrixXd mx_phyiscs_basis, - double threshold) { +inline std::pair +CalculateTruncatedPCARotation(Eigen::MatrixXd mx_syst_basis, double threshold) { + + int nsyst_params = int(mx_syst_basis.rows()); // a covariance is real symmetric, so self adjoint - Eigen::SelfAdjointEigenSolver EigenSolver(mx_phyiscs_basis); + Eigen::SelfAdjointEigenSolver EigenSolver(mx_syst_basis); if (EigenSolver.info() != Eigen::Success) { MACH3LOG_ERROR("Failed to eigen decompose matrix."); @@ -31,10 +33,7 @@ inline Eigen::MatrixXd CalculateTruncatedPCARotation(Eigen::MatrixXd mx_phyiscs_ evals.emplace_back(i, eigen_val[i]); } - std::cout << "PCA:\n evals: " << eigen_val << "\n evects:\n" - << eigen_vect << std::endl; - - // sorting is not strictly neccessary but orders the orthogonal parameter + // sorting is not strictly neccessary but orders the pc parameter // basis approximately by uncertainty. std::sort( evals.rbegin(), evals.rend(), @@ -43,32 +42,40 @@ inline Eigen::MatrixXd CalculateTruncatedPCARotation(Eigen::MatrixXd mx_phyiscs_ }); double evalsum = 0; - int northo_parameters = 0; + int npc_parameters = 0; - double trace = mx_phyiscs_basis.trace(); + double trace = mx_syst_basis.trace(); for (auto &[i, ev] : evals) { - if (std::fabs(ev / trace) > threshold) { - evalsum += std::fabs(ev); - northo_parameters++; + if ((ev / trace) > threshold) { + evalsum += ev; + npc_parameters++; } else { break; } } - // rows of this matrix correspond to eigenvectors of the input matrix - // we can go from parameters defined in the orthogonal basis back to the - // 'physics' parameters by RowVectOfOrthoParamVals * pca.ortho_to_physics - Eigen::MatrixXd ortho_to_physics = - Eigen::MatrixXd::Zero(northo_parameters, mx_phyiscs_basis.rows()); - - for (int i = 0; i < northo_parameters; ++i) { - for (int j = 0; j < mx_phyiscs_basis.rows(); ++j) { - ortho_to_physics.row(i)[j] = - eigen_vect.col(evals[i].first)[evals[j].first] * - std::sqrt(evals[i].second); + // cols of this matrix correspond to eigenvectors of the input matrix + // we can rotate from the systematic basis to the pc basis by: + // TruncatedEigenVectorMatrix * ColVecPCParams = ColVecSystParams + Eigen::MatrixXd TruncatedEigenVectorMatrix = + Eigen::MatrixXd::Zero(nsyst_params, npc_parameters); + Eigen::MatrixXd TruncatedsqrtEigenValueMatrix = + Eigen::MatrixXd::Zero(npc_parameters, npc_parameters); + + for (int i = 0; i < npc_parameters; ++i) { + TruncatedEigenVectorMatrix.col(i) = eigen_vect.col(evals[i].first); + + // bias eigen vectors towards being in the positive direction relative to + // other parameters + if (TruncatedEigenVectorMatrix.col(i).sum() < 0) { + TruncatedEigenVectorMatrix.col(i).array() *= -1; } + + TruncatedsqrtEigenValueMatrix(i, i) = std::sqrt(evals[i].second); } - return ortho_to_physics; + return {TruncatedEigenVectorMatrix * TruncatedsqrtEigenValueMatrix, + TruncatedsqrtEigenValueMatrix.inverse() * + TruncatedEigenVectorMatrix.transpose()}; } diff --git a/Parameters/ParameterHandlerBase.cpp b/Parameters/ParameterHandlerBase.cpp index 5f6fd1449..f5a0128d3 100644 --- a/Parameters/ParameterHandlerBase.cpp +++ b/Parameters/ParameterHandlerBase.cpp @@ -1,340 +1,114 @@ #include "Parameters/ParameterHandlerBase.h" -#include -ParameterHandlerBase::ParameterHandlerBase() { - pca.enabled = false; - special_proposal.enabled = false; - rng.gaus = std::normal_distribution(0); - rng.unif = std::uniform_real_distribution(0, 1); - settings.use_adaptive = false; - settings.PrintLength = 35; -} +#include // ******************************************** -ParameterHandlerBase::ParameterHandlerBase(std::string const &name, - std::string const &file) - : ParameterHandlerBase() { +ParameterHandlerBase::ParameterHandlerBase(std::string name, std::string file, + double, int, int) + : inputFile(file) { // ******************************************** + MACH3LOG_DEBUG("Constructing instance of ParameterHandler"); - SetName(name); - config.inputFiles = { - file, - }; - - // Set the covariance matrix from input ROOT file (e.g. flux, ND280, NIWG) - TFile infile(file.c_str(), "READ"); - if (infile.IsZombie()) { - MACH3LOG_ERROR("Could not open input covariance ROOT file {} !!!", file); - MACH3LOG_ERROR("Was about to retrieve matrix with name {}", name); - throw MaCh3Exception(__FILE__, __LINE__); - } + parlist = ParameterList::MakeFromTMatrix(name, file); + current = parlist.params.prefit; + proposed = parlist.params.prefit; - auto *CovMat = infile.Get(name.c_str()); - - if (!CovMat) { - MACH3LOG_ERROR("Could not find covariance matrix name {} in file {}", name, - file); - MACH3LOG_ERROR("Are you really sure {} exists in the file?", name); - throw MaCh3Exception(__FILE__, __LINE__); - } + covMatrix = new TMatrixDSym(int(parlist.params.covariance.rows()), + int(parlist.params.covariance.cols())); + M3::EigenToROOT(parlist.params.covariance, *covMatrix); + invCovMatrix = new TMatrixDSym(int(parlist.params.covariance.rows()), + int(parlist.params.covariance.cols())); + M3::EigenToROOT(parlist.params.inv_covariance, *invCovMatrix); - params.covariance = M3::MakeMatrixPosDef(M3::ROOTToEigen(*CovMat)); + proposer = parlist.MakeProposer(); - MACH3LOG_INFO("Created covariance matrix named: {}", GetName()); - MACH3LOG_INFO("from file: {}", file); + throwMatrix = new TMatrixDSym(int(proposer.params.proposal.rows()), + int(proposer.params.proposal.cols())); + M3::EigenToROOT(proposer.params.proposal, *throwMatrix); } - // ******************************************** ParameterHandlerBase::ParameterHandlerBase( - const std::vector &YAMLFiles, std::string const &name) - : ParameterHandlerBase() { + const std::vector &YAMLFile, std::string name, + double threshold, int FirstPCA, int LastPCA) + : matrixName(name), inputFile(YAMLFile[0].c_str()) { // ******************************************** + MACH3LOG_INFO("Constructing instance of ParameterHandler using"); - SetName(name); - config.inputFiles = YAMLFiles; - - MACH3LOG_INFO("Constructing instance of ParameterHandler using: "); - for (auto const &yf : config.inputFiles) { - MACH3LOG_INFO(" {}", yf); + for (unsigned int i = 0; i < YAMLFile.size(); i++) { + MACH3LOG_INFO("{}", YAMLFile[i]); } - MACH3LOG_INFO("as an input."); + MACH3LOG_INFO("as an input"); - config.YAMLDoc["Systematics"] = YAML::Node(YAML::NodeType::Sequence); - for (auto const &yfn : config.inputFiles) { - for (const auto &node : M3OpenConfig(yfn)["Systematics"]) { - config.YAMLDoc["Systematics"].push_back(node); - } + bool pca = true; + if (threshold < 0 || threshold >= 1) { + MACH3LOG_INFO("Principal component analysis but given the threshold for " + "the principal components to be less than 0, or greater than " + "(or equal to) 1. This will not work"); + MACH3LOG_INFO("Please specify a number between 0 and 1"); + MACH3LOG_INFO("You specified: "); + MACH3LOG_INFO("Am instead calling the usual non-PCA constructor..."); + pca = false; } - std::map> parameter_correlations; - - std::vector param_infos; - - // ETA - read in the systematics. - for (auto const &node : config.YAMLDoc["Systematics"]) { - int param_id = int(param_infos.size()); - - auto const &pardef = node["Systematic"]; - - auto fancy_name = - Get(pardef["Names"]["FancyName"], __FILE__, __LINE__); - auto prefit = Get(pardef["ParameterValues"]["PreFitValue"], - __FILE__, __LINE__); + parlist = ParameterList::MakeFromYAML(YAMLFile); + current = parlist.params.prefit; + proposed = parlist.params.prefit; - auto error = Get(pardef["Error"], __FILE__, __LINE__); + covMatrix = new TMatrixDSym(int(parlist.params.covariance.rows()), + int(parlist.params.covariance.cols())); + M3::EigenToROOT(parlist.params.covariance, *covMatrix); + invCovMatrix = new TMatrixDSym(int(parlist.params.covariance.rows()), + int(parlist.params.covariance.cols())); + M3::EigenToROOT(parlist.params.inv_covariance, *invCovMatrix); - if (error <= 0) { - MACH3LOG_ERROR("Error for param {}({}) is negative and equal to {}", - fancy_name, param_id, error); - throw MaCh3Exception(__FILE__, __LINE__); - } - - auto stepscale = - Get(pardef["StepScale"]["MCMC"], __FILE__, __LINE__); - - auto bounds = GetBounds(pardef["ParameterBounds"]); - - auto flatprior = - GetFromManager(pardef["FlatPrior"], false, __FILE__, __LINE__); - - auto samplenames = GetFromManager>( - pardef["SampleNames"], {}, __FILE__, __LINE__); - - // Allow to fix param, this setting should be used only for params which are - // permanently fixed like baseline, please use global config for fixing - // param more flexibly - auto fixed = - GetFromManager(pardef["FixParam"], false, __FILE__, __LINE__); - - if (pardef["SpecialProposal"]) { - EnableSpecialProposal(pardef["SpecialProposal"], param_id); - } - - if (pardef["Correlations"]) { - for (auto const &corrn : pardef["Correlations"]) { - for (auto const &corr : corrn) { - parameter_correlations[param_id][corr.first.as()] = - corr.second.as(); - } - } - } - - param_infos.emplace_back(ParamInfo{fancy_name, - fancy_name, - prefit, - error, - stepscale, - {bounds[0], bounds[1]}, - flatprior, - !fixed, - samplenames}); + if (pca) { + ConstructPCA(threshold, FirstPCA, LastPCA); } - AddParameters(param_infos); - - for (auto const &[paramid, correlations] : parameter_correlations) { - SetParameterAllCorrelations(paramid, correlations); - } + proposer = parlist.MakeProposer(); - params.covariance = M3::MakeMatrixPosDef(params.covariance); - - Tunes = ParameterTunes(config.YAMLDoc["Systematics"]); - - MACH3LOG_INFO("Created covariance matrix from files: "); - for (const auto &file : config.inputFiles) { - MACH3LOG_INFO("{} ", file); - } - MACH3LOG_INFO("----------------"); - MACH3LOG_INFO("Found {} systematics parameters in total", - params.prefit.size()); - MACH3LOG_INFO("----------------"); + throwMatrix = new TMatrixDSym(int(proposer.params.proposal.rows()), + int(proposer.params.proposal.cols())); + M3::EigenToROOT(proposer.params.proposal, *throwMatrix); } // ******************************************** -void ParameterHandlerBase::ConstructPCA(const double threshold, int first, - int last) { +void ParameterHandlerBase::ConstructPCA(const double threshold, + int FirstPCAdpar, int LastPCAdpar) { // ******************************************** - if (settings.use_adaptive) { - MACH3LOG_ERROR("Adaption has been enabled and now trying to enable PCA. " - "Right now both configuration don't work with each other"); - throw MaCh3Exception(__FILE__, __LINE__); - } - - block_indices[0] = first; - block_indices[1] = last; - - int nphysics_parameters = (last - first) + 1; - - double block_trace = - params.covariance.block(first, first, last - first + 1, last - first + 1) - .trace(); - - // a covariance is real symmetric, so self adjoint - Eigen::SelfAdjointEigenSolver EigenSolver( - params.covariance.block(first, first, nphysics_parameters, - nphysics_parameters)); - Eigen::VectorXd eigen_val = EigenSolver.eigenvalues(); - Eigen::MatrixXd eigen_vect = EigenSolver.eigenvectors(); - std::vector> evals; - for (int i = 0; i < eigen_val.size(); ++i) { - evals.emplace_back(i, eigen_val[i]); - } - std::sort( - evals.begin(), evals.end(); - [](std::pair const &l, std::pair const &r) { - return l.second < r.second; - }); - - double evsum = 0; - int northo_parameters = 0; - - for (auto &[i, ev] : evals) { - if (std::fabs(ev) > threshold) { - evsum += std::fabs(ev); - northo_parameters++; - } else { - break; - } - } - - //rows of this matrix correspond to eigenvectors of the input matrix - // we can go from parameters defined in the orthogonal basis back to the - // 'physics' parameters by RowVectOfOrthoParamVals * pca.ortho_to_physics - pca.ortho_to_physics = - Eigen::MatrixXd::Zero(evals.size(), nphysics_parameters); - - for (size_t i = 0; i < northo_parameters; ++i) { - for (size_t j = 0; j < nphysics_parameters; ++j) { - pca.ortho_to_physics.row(i)[j] = - eigen_vect.col(evals[i].first)[evals[j].first] * - std::sqrt(evals[i].second); - } - } - - MACH3LOG_INFO("Threshold of {} on eigen values, kept {}/{} for a total " - "variance of {}/{}", - threshold, nkept, evals.size(), evsum, block_trace); + parlist.ConstructTruncatedPCA(threshold, FirstPCAdpar, LastPCAdpar); } // ******************************************** -void ParameterHandlerBase::EnableSpecialProposal(const YAML::Node ¶m, - const int Index) { +// Set the covariance matrix for this class +void ParameterHandlerBase::SetCovMatrix(TMatrixDSym *) { // ******************************************** - special_proposal.enabled = true; - - bool CircEnabled = false; - bool FlipEnabled = false; - - if (param["CircularBounds"]) { - CircEnabled = true; - } - - if (param["FlipParameter"]) { - FlipEnabled = true; - } - - if (!CircEnabled && !FlipEnabled) { - MACH3LOG_ERROR("None of Special Proposal were enabled even though param " - "{}, has SpecialProposal entry in Yaml", - GetParFancyName(Index)); - throw MaCh3Exception(__FILE__, __LINE__); - } - - if (CircEnabled) { - auto bounds = Get>(param["CircularBounds"], - __FILE__, __LINE__); - // KS: Make sure circular bounds are within physical bounds. If we are - // outside of physics bound MCMC will never explore such phase space region - if (bounds.first < params.lowbound[Index] || - bounds.second > params.upbound[Index]) { - MACH3LOG_ERROR("Circular bounds [{}, {}] for parameter {} exceed " - "physical bounds [{}, {}]", - bounds.first, bounds.second, GetParFancyName(Index), - params.lowbound[Index], params.upbound[Index]); - throw MaCh3Exception(__FILE__, __LINE__); - } - - special_proposal.circ_bounds.emplace_back( - std::make_tuple(Index, bounds.first, bounds.second)); - - MACH3LOG_INFO( - "Enabling CircularBounds for parameter {} with range [{}, {}]", - GetParFancyName(Index), bounds.first, bounds.second); - } - - if (FlipEnabled) { - special_proposal.flips.emplace_back(std::make_pair( - Index, Get(param["FlipParameter"], __FILE__, __LINE__))); - MACH3LOG_INFO("Enabling Flipping for parameter {} with value {}", - GetParFancyName(Index), special_proposal.flips.back().second); - } - - if (CircEnabled && FlipEnabled) { - - const double fp = special_proposal.flips.back().second; - const double low = std::get<1>(special_proposal.circ_bounds.back()); - const double high = std::get<2>(special_proposal.circ_bounds.back()); - - if (fp < low || fp > high) { - MACH3LOG_ERROR("FlipParameter value {} for parameter {} is outside the " - "CircularBounds [{}, {}]", - fp, GetParFancyName(Index), low, high); - throw MaCh3Exception(__FILE__, __LINE__); - } - - // Sanity check: ensure flipping any x in [low, high] keeps the result in - // [low, high] - const double flipped_low = 2 * fp - low; - const double flipped_high = 2 * fp - high; - const double min_flip = std::min(flipped_low, flipped_high); - const double max_flip = std::max(flipped_low, flipped_high); - - if (min_flip < low || max_flip > high) { - MACH3LOG_ERROR("Flipping about point {} for parameter {} would leave " - "circular bounds [{}, {}]", - special_proposal.flips.back().second, - GetParFancyName(Index), low, high); - throw MaCh3Exception(__FILE__, __LINE__); - } - } + throw std::runtime_error("ParameterHandlerBase::SetCovMatrix Removed"); } // ******************************************** -// Set the covariance matrix for this class -void ParameterHandlerBase::SetCovMatrix(TMatrixDSym *cov) { +// Set all the covariance matrix parameters to a user-defined value +// Might want to split this +void ParameterHandlerBase::SetPar(int i, double val) { // ******************************************** - if (cov == nullptr) { - MACH3LOG_ERROR("Could not find covariance matrix you provided to {}", - __func__); - throw MaCh3Exception(__FILE__, __LINE__); - } - if (cov->GetNrows() != params.prefit.size()) { - MACH3LOG_ERROR( - "Passed covariance matrix size {0}x{0}, but have {1} parameters.", - cov->GetNrows(), params.prefit.size()); - throw MaCh3Exception(__FILE__, __LINE__); - } + current[i] = val; + proposed[i] = val; - params.covariance = M3::ROOTToEigen(*cov); - params.inv_covariance = params.covariance.inverse(); + MACH3LOG_INFO("Overriding {}: ", GetParName(i)); + MACH3LOG_INFO("_fPropVal ({}), _fCurrVal ({}), _fPreFitValue ({}) to ({})", + proposed[i], current[i], parlist.params.prefit[i], val); - SetThrowMatrix(cov); + parlist.RotateSystematicParameterValuesToPCBasis(current, + proposer.params.current); + parlist.RotateSystematicParameterValuesToPCBasis(proposed, + proposer.params.proposed); } // ******************************************** -// Set all the covariance matrix parameters to a user-defined value -// Might want to split this -void ParameterHandlerBase::SetPar(int i, double val) { +std::vector ParameterHandlerBase::GetProposed() const { // ******************************************** - MACH3LOG_INFO("Overriding {}: ", GetParName(i)); - MACH3LOG_INFO( - "steps.proposed ({}), steps.current ({}), params.prefit ({}) to ({})", - steps.proposed[i], steps.current[i], params.prefit[i], val); - - steps.proposed[i] = val; - steps.current[i] = val; - params.prefit[i] = val; + return M3::EigenToStdVector(proposer.params.proposed); } // ************************************* @@ -343,37 +117,29 @@ void ParameterHandlerBase::SetPar(int i, double val) { void ParameterHandlerBase::ThrowParameters() { // ************************************* - // First draw a new random_vector - Randomize(); - - throws.values = steps.l_proposal * throws.random_vector; - - steps.proposed = params.prefit + throws.values; - - for (int i = 0; i < throws.values.size(); ++i) { - - int tries = 0; - while ((steps.proposed[i] < params.lowbound[i]) || - (steps.proposed[i] >= params.upbound[i])) { - steps.proposed[i] = - params.prefit[i] + (steps.l_proposal.col(i) * rng.gaus(rng.e1))[i]; - tries++; - if (tries > 10000) { - // KS: Since we are multithreading there is danger that those messages - // will be all over the place, small price to pay for faster code - MACH3LOG_WARN("Tried {} times to throw parameter {} but failed", tries, - i); - MACH3LOG_WARN("Matrix: {}", settings.name); - MACH3LOG_WARN("Param: {}", params.name[i]); - MACH3LOG_WARN("Setting steps.proposed: {} to {}", steps.proposed[i], - params.prefit[i]); - MACH3LOG_WARN("I live at {}:{}", __FILE__, __LINE__); - steps.proposed[i] = params.prefit[i]; - } + int tries = 0; + proposer.Propose(); + while (CheckBounds()) { + proposer.Propose(); + + tries++; + if (tries > 10000) { + // KS: Since we are multithreading there is danger that those messages + // will be all over the place, small price to pay for faster code + MACH3LOG_ERROR( + "Tried {} times to throw parameters within bounds but failed.", + tries); + MACH3LOG_ERROR("Matrix: {}", GetName()); + throw MaCh3Exception(__FILE__, __LINE__); } - - steps.current = steps.proposed; } + + proposer.Accept(); + + parlist.RotatePCParameterValuesToSystematicBasis(proposer.params.current, + current); + parlist.RotatePCParameterValuesToSystematicBasis(proposer.params.proposed, + proposed); } // ************************************* @@ -383,39 +149,49 @@ void ParameterHandlerBase::RandomConfiguration() { // ************************************* // Have the 1 sigma for each parameter in each covariance class, sweet! // Don't want to change the prior array because that's what determines our - // likelihood Want to change the steps.proposed, steps.current, params.prefit - // params.prefit and the others will already be set + // likelihood Want to change the _fPropVal, _fCurrVal, _fPreFitValue + // _fPreFitValue and the others will already be set - for (int i = 0; i < params.prefit.size(); ++i) { + for (int i = 0; i < parlist.NumSystematicBasisParameters(); ++i) { // Check if parameter is fixed first: if so don't randomly throw if (IsParameterFixed(i)) { continue; } // Check that the sigma range is larger than the parameter range // If not, throw in the valid parameter range instead - double throwrange = std::min(params.upbound[i] - params.lowbound[i], - std::sqrt(params.covariance(i, i))); + const double paramrange = + parlist.params.upbound[i] - parlist.params.lowbound[i]; + const double sigma = parlist.params.error[i]; + double throwrange = sigma; + if (paramrange < sigma) { + throwrange = paramrange; + } - steps.proposed[i] = params.prefit[i] + rng.gaus(rng.e1) * throwrange; + current[i] = parlist.params.prefit[i] + + proposer.rng.gaus(proposer.rng.e1) * throwrange; // Try again if we the initial parameter proposal falls outside of the range // of the parameter - int nthrows = 0; - while ((steps.proposed[i] > params.upbound[i]) || - (steps.proposed[i] < params.lowbound[i])) { - if (nthrows > 1000) { - MACH3LOG_WARN("Tried {} times to throw parameter {} but failed", - nthrows, i); - MACH3LOG_WARN("Matrix: {}", settings.name); - MACH3LOG_WARN("Param: {}", params.name[i]); + int throws = 0; + while (current[i] > parlist.params.upbound[i] || + current[i] < parlist.params.lowbound[i]) { + if (throws > 1000) { + MACH3LOG_WARN("Tried {} times to throw parameter {} but failed", throws, + i); + MACH3LOG_WARN("Matrix: {}", GetName()); + MACH3LOG_WARN("Param: {}", GetParName(i)); throw MaCh3Exception(__FILE__, __LINE__); } - steps.proposed[i] = params.prefit[i] + rng.gaus(rng.e1) * throwrange; - nthrows++; + current[i] = parlist.params.prefit[i] + + proposer.rng.gaus(proposer.rng.e1) * throwrange; + throws++; } - MACH3LOG_INFO("Setting current step in {} param {} = {} from {}", - settings.name, i, steps.proposed[i], steps.current[i]); - steps.current[i] = steps.proposed[i]; + MACH3LOG_INFO("Setting current step in {} param {}, {} = {} from {}", + GetName(), i, GetParName(i), current[i], current[i]); } + parlist.RotateSystematicParameterValuesToPCBasis(current, + proposer.params.proposed); + proposer.Accept(); + proposed = current; } // ************************************* @@ -423,111 +199,83 @@ void ParameterHandlerBase::RandomConfiguration() { void ParameterHandlerBase::SetSingleParameter(const int parNo, const double parVal) { // ************************************* - steps.current[parNo] = parVal; - steps.proposed[parNo] = parVal; - MACH3LOG_DEBUG("Setting {} (parameter {}) to {})", GetParName(parNo), parNo, - parVal); + SetPar(parNo, parVal); } // ******************************************** void ParameterHandlerBase::SetParCurrProp(const int parNo, const double parVal) { // ******************************************** - SetSingleParameter(parNo, parVal); + SetPar(parNo, parVal); } -// ************************************************ -// Propose a step for the set of systematics parameters this covariance class -// holds -void ParameterHandlerBase::ProposeStep() { - // ************************************************ - // Make the random numbers for the step proposal - Randomize(); - CorrelateSteps(); - - // KS: According to Dr Wallace we update using previous not proposed step - // this way we do special proposal after adaptive after. - // This way we can shortcut and skip rest of proposal - if (!special_proposal.enabled) { - return; - } - - SpecialStepProposal(); +void ParameterHandlerBase::SetParProp(const int i, const double val) { + proposed[i] = val; + parlist.RotateSystematicParameterValuesToPCBasis(proposed, + proposer.params.proposed); } -// ************************************************ -void ParameterHandlerBase::SpecialStepProposal() { - // ************************************************ - /// @warning KS: Following Asher comment we do "Step->Circular Bounds->Flip" - - // HW It should now automatically set dcp to be with [-pi, pi] - for (auto const &[idx, low, up] : special_proposal.circ_bounds) { - if (IsParameterFixed(idx)) { - continue; - } - if (steps.proposed[idx] > up) { - steps.proposed[idx] = low + std::fmod(steps.proposed[idx] - up, up - low); - } else if (steps.proposed[idx] < low) { - steps.proposed[idx] = up - std::fmod(low - steps.proposed[idx], up - low); - } - } - - // Okay now we've done the standard steps, we can add in our nice flips - // hierarchy flip first - for (auto const &[idx, pivot] : special_proposal.flips) { - if (IsParameterFixed(idx)) { - continue; - } - if (rng.unif(rng.e1) < 0.5) { - steps.proposed[idx] = 2 * pivot - steps.proposed[idx]; - } - } +void ParameterHandlerBase::SetRandomThrow(const int, const double) { + // ******************************************** + throw std::runtime_error("ParameterHandlerBase::SetRandomThrow Removed"); } - -// ************************************************ -// "Randomize" the parameters in the covariance class for the proposed step -// Used the proposal kernel and the current parameter value to set proposed step -// Also get a new random number for the randParams -void ParameterHandlerBase::Randomize() _noexcept_ { - // ************************************************ - throws.random_vector = - Eigen::VectorXd::NullaryExpr(params.prefit.size(), [&](int i) { - return params.isfree[i] ? rng.gaus(rng.e1) : 0; - }); +double ParameterHandlerBase::GetRandomThrow(const int) const { + // ******************************************** + throw std::runtime_error("ParameterHandlerBase::GetRandomThrow Removed"); } // ************************************************ -// Correlate the steps by setting the proposed step of a parameter to its -// current value + some correlated throw -void ParameterHandlerBase::CorrelateSteps() _noexcept_ { +// Propose a step for the set of systematics parameters this covariance class +// holds +void ParameterHandlerBase::ProposeStep() { // ************************************************ - - Randomize(); - - // array makes the multiplaction by steps.scale component-wise rather than - // vector-ie - steps.proposed = - steps.current + ((steps.l_proposal * throws.random_vector).array() * - steps.scale.array() * steps.global_scale) - .matrix(); + proposer.Propose(); + parlist.RotatePCParameterValuesToSystematicBasis(proposer.params.proposed, + proposed); } + // ******************************************** // Update so that current step becomes the previously proposed step void ParameterHandlerBase::AcceptStep() _noexcept_ { // ******************************************** - steps.current = steps.proposed; + proposer.Accept(); + parlist.RotatePCParameterValuesToSystematicBasis(proposer.params.current, + current); +} - if (settings.use_adaptive) { - AdaptiveHandler.IncrementAcceptedSteps(); - } +// ******************************************** +// Throw the proposed parameter by mag sigma +// Should really just have the user specify this throw by having argument double +void ParameterHandlerBase::ThrowParProp(const double mag) { + // ******************************************** + Eigen::VectorXd new_proposed = + ((proposer.Propose() - proposer.params.current) * mag) + + proposer.params.current; + // probably fine but avoid ailiasing for now + proposer.params.proposed = new_proposed; + parlist.RotatePCParameterValuesToSystematicBasis(new_proposed, proposed); } +// ******************************************** +// Helper function to throw the current parameter by mag sigmas +// Can study bias in MCMC with this; put different starting parameters +void ParameterHandlerBase::ThrowParCurr(const double mag) { + // ******************************************** + + Eigen::VectorXd curr_proposed = proposer.params.proposed; + Eigen::VectorXd new_curr = + ((proposer.Propose() - proposer.params.current) * mag); + proposer.params.current = new_curr; + // probably fine but avoid ailiasing for now + proposer.params.proposed = curr_proposed; + parlist.RotatePCParameterValuesToSystematicBasis(new_curr, current); +} // ******************************************** // Function to print the prior values void ParameterHandlerBase::PrintNominal() const { // ******************************************** MACH3LOG_INFO("Prior values for {} ParameterHandler:", GetName()); - for (int i = 0; i < int(params.name.size()); i++) { + for (int i = 0; i < parlist.NumSystematicBasisParameters(); i++) { MACH3LOG_INFO(" {} {} ", GetParFancyName(i), GetParInit(i)); } } @@ -536,12 +284,19 @@ void ParameterHandlerBase::PrintNominal() const { // Function to print the prior, current and proposed values void ParameterHandlerBase::PrintNominalCurrProp() const { // ******************************************** + + parlist.RotatePCParameterValuesToSystematicBasis(proposer.params.current, + current); + + parlist.RotatePCParameterValuesToSystematicBasis(proposer.params.proposed, + proposed); + MACH3LOG_INFO("Printing parameters for {}", GetName()); MACH3LOG_INFO("{:<30} {:<10} {:<10} {:<10}", "Name", "Prior", "Current", "Proposed"); - for (int i = 0; i < params.prefit.size(); ++i) { + for (int i = 0; i < parlist.NumSystematicBasisParameters(); ++i) { MACH3LOG_INFO("{:<30} {:<10.2f} {:<10.2f} {:<10.2f}", GetParFancyName(i), - params.prefit[i], steps.current[i], steps.proposed[i]); + parlist.params.prefit[i], proposed[i], current[i]); } } @@ -551,21 +306,23 @@ void ParameterHandlerBase::PrintNominalCurrProp() const { // given parameter // true = don't evaluate likelihood (so run without a prior) // false = evaluate likelihood (so run with a prior) - -double ParameterHandlerBase::CalcLikelihood() const _noexcept_ { +double ParameterHandlerBase::CalcLikelihood() _noexcept_ { // ******************************************** - // filter out parameters with flat priors from the pull - auto diff = - params.flatprior.select(Eigen::VectorXd::Zero(params.prefit.size()), - steps.proposed - params.prefit); - return diff.transpose() * params.inv_covariance * diff; + parlist.RotatePCParameterValuesToSystematicBasis(proposer.params.proposed, + proposed); + + return parlist.Chi2(proposed); } // ******************************************** int ParameterHandlerBase::CheckBounds() const _noexcept_ { // ******************************************** - return int(((steps.proposed.array() < params.lowbound.array()) || - (steps.proposed.array() >= params.upbound.array())) + + parlist.RotatePCParameterValuesToSystematicBasis(proposer.params.proposed, + proposed); + + return int(((proposed.array() < parlist.params.lowbound.array()) || + (proposed.array() >= parlist.params.upbound.array())) .count()); } @@ -581,33 +338,47 @@ double ParameterHandlerBase::GetLikelihood() { return CalcLikelihood(); } +double ParameterHandlerBase::GetInvCovMatrix(const int, const int) const { + + throw std::runtime_error("ParameterHandlerBase::GetInvCovMatrix Removed"); +} + +double ParameterHandlerBase::GetCorrThrows(const int) const { + + throw std::runtime_error("ParameterHandlerBase::GetCorrThrows Removed"); +} + +bool ParameterHandlerBase::GetFlatPrior(const int i) const { + int propidx = parlist.SystematicParameterIndexToPCIndex(i); + if (propidx != ParameterList::ParameterInPCABlock) { + return parlist.params.flatprior[i]; + } else { + MACH3LOG_ERROR("You are trying Fix parameter {}, {}, which has been PCA'd.", + i, GetParName(i)); + throw MaCh3Exception(__FILE__, __LINE__); + } +} + // ******************************************** // Sets the proposed parameters to the prior values void ParameterHandlerBase::SetParameters(const std::vector &pars) { // ******************************************** // If empty, set the proposed to prior if (pars.empty()) { - steps.proposed = params.prefit; + // For xsec this means setting to the prior (because prior is the prior) + parlist.RotateSystematicParameterValuesToPCBasis(parlist.params.prefit, + proposer.params.proposed); // If not empty, set the parameters to the specified } else { - if (pars.size() != size_t(steps.proposed.size())) { + if (int(pars.size()) != parlist.NumSystematicBasisParameters()) { MACH3LOG_ERROR("Parameter arrays of incompatible size! Not changing " "parameters! {} has size {} but was expecting {}", - settings.name, pars.size(), steps.proposed.size()); + GetName(), pars.size(), + parlist.NumSystematicBasisParameters()); throw MaCh3Exception(__FILE__, __LINE__); } - for (int i = 0; i < int(pars.size()); i++) { - // Make sure that you are actually passing a number to set the parameter - // to - if (std::isnan(pars[i])) { - MACH3LOG_ERROR("Trying to set parameter value to a nan for parameter " - "{} in matrix {}. This will not go well!", - GetParName(i), settings.name); - throw MaCh3Exception(__FILE__, __LINE__); - } else { - steps.proposed[i] = pars[i]; - } - } + parlist.RotateSystematicParameterValuesToPCBasis(M3::StdVectorToEigen(pars), + proposer.params.proposed); } } @@ -615,27 +386,17 @@ void ParameterHandlerBase::SetParameters(const std::vector &pars) { void ParameterHandlerBase::SetBranches(TTree &tree, bool SaveProposal) { // ******************************************** // loop over parameters and set a branch - for (int i = 0; i < int(params.name.size()); ++i) { - tree.Branch(params.name[i].c_str(), steps.current.data() + i, - Form("%s/D", params.name[i].c_str())); + for (int i = 0; i < parlist.NumSystematicBasisParameters(); ++i) { + tree.Branch(GetParName(i).c_str(), ¤t.data()[i], + Form("%s/D", GetParName(i).c_str())); } - // // When running PCA, also save PCA parameters - // if (pca.enabled) { - // PCAObj.SetBranches(tree, SaveProposal, _fNames); - // } if (SaveProposal) { // loop over parameters and set a branch - for (int i = 0; i < int(params.name.size()); ++i) { - tree.Branch(Form("%s_Prop", params.name[i].c_str()), - steps.proposed.data() + i, - Form("%s_Prop/D", params.name[i].c_str())); + for (int i = 0; i < parlist.NumSystematicBasisParameters(); ++i) { + tree.Branch(Form("%s_Prop", GetParName(i).c_str()), &proposed.data()[i], + Form("%s_Prop/D", GetParName(i).c_str())); } } - if (settings.use_adaptive && AdaptiveHandler.GetUseRobbinsMonro()) { - tree.Branch(Form("GlobalStepScale_%s", GetName().c_str()), - &steps.global_scale, - Form("GlobalStepScale_%s/D", GetName().c_str())); - } } // ******************************************** @@ -643,43 +404,47 @@ void ParameterHandlerBase::SetStepScale(const double scale, const bool verbose) { // ******************************************** if (scale <= 0) { - MACH3LOG_ERROR("Invalid step scale {}", scale); + MACH3LOG_ERROR( + "You are trying so set StepScale to 0 or negative this will not work"); throw MaCh3Exception(__FILE__, __LINE__); } if (verbose) { MACH3LOG_INFO("{} setStepScale() = {}", GetName(), scale); - const double SuggestedScale = 2.38 / std::sqrt(steps.scale.size()); - if ((std::fabs(scale - SuggestedScale) / SuggestedScale) > 1) { + const double SuggestedScale = + 2.38 / std::sqrt(parlist.NumPCBasisParameters()); + if (std::fabs(scale - SuggestedScale) / SuggestedScale > 1) { MACH3LOG_WARN( "Defined Global StepScale is {}, while suggested suggested {}", scale, SuggestedScale); } } - steps.global_scale = scale; + proposer.params.global_scale = scale; } // ******************************************** int ParameterHandlerBase::GetParIndex(const std::string &name) const { // ******************************************** - auto idx = std::find(params.name.begin(), params.name.end(), name); - if (idx == params.name.end()) { - MACH3LOG_ERROR("No parameter with the name {} exists.", name); - throw MaCh3Exception(__FILE__, __LINE__); - } - return int(std::distance(params.name.begin(), idx)); + return parlist.FindParameter(name); } // ******************************************** void ParameterHandlerBase::SetFixAllParameters() { // ******************************************** - params.isfree = Eigen::VectorXi::Zero(params.prefit.size()); + proposer.params.isfree = Eigen::ArrayXi::Ones(proposer.NumParameters()); } // ******************************************** void ParameterHandlerBase::SetFixParameter(const int i) { // ******************************************** - params.isfree[i] = 0; + int propidx = parlist.SystematicParameterIndexToPCIndex(i); + if (propidx != ParameterList::ParameterInPCABlock) { + proposer.params.isfree[propidx] = false; + } else { + MACH3LOG_ERROR("You are trying Fix parameter {}, {}, which has been PCA'd.", + i, GetParName(i)); + throw MaCh3Exception(__FILE__, __LINE__); + } } // ******************************************** @@ -691,13 +456,21 @@ void ParameterHandlerBase::SetFixParameter(const std::string &name) { // ******************************************** void ParameterHandlerBase::SetFreeAllParameters() { // ******************************************** - params.isfree = Eigen::VectorXi::Ones(params.prefit.size()); + proposer.params.isfree = Eigen::ArrayXi::Ones(proposer.NumParameters()); } // ******************************************** void ParameterHandlerBase::SetFreeParameter(const int i) { // ******************************************** - params.isfree[i] = 1; + int propidx = parlist.SystematicParameterIndexToPCIndex(i); + if (propidx != ParameterList::ParameterInPCABlock) { + proposer.params.isfree[propidx] = true; + } else { + MACH3LOG_ERROR( + "You are trying Free parameter {}, {}, which has been PCA'd.", i, + GetParName(i)); + throw MaCh3Exception(__FILE__, __LINE__); + } } // ******************************************** @@ -709,62 +482,94 @@ void ParameterHandlerBase::SetFreeParameter(const std::string &name) { // ******************************************** void ParameterHandlerBase::ToggleFixAllParameters() { // ******************************************** - throw std::runtime_error("Don't Toggle parameters"); + throw std::runtime_error( + "ParameterHandlerBase::ToggleFixAllParameters Removed"); } // ******************************************** void ParameterHandlerBase::ToggleFixParameter(const int) { // ******************************************** - throw std::runtime_error("Don't Toggle parameters"); + throw std::runtime_error("ParameterHandlerBase::ToggleFixParameter Removed"); } // ******************************************** void ParameterHandlerBase::ToggleFixParameter(const std::string &) { // ******************************************** - throw std::runtime_error("Don't Toggle parameters"); + throw std::runtime_error("ParameterHandlerBase::ToggleFixParameter Removed"); +} + +bool ParameterHandlerBase::IsParameterFixed(const int) const { + throw std::runtime_error("ParameterHandlerBase::IsParameterFixed Removed"); } // ******************************************** -bool ParameterHandlerBase::IsParameterFixed(const std::string &name) const { +bool ParameterHandlerBase::IsParameterFixed(const std::string &) const { // ******************************************** - return IsParameterFixed(GetParIndex(name)); + throw std::runtime_error("ParameterHandlerBase::IsParameterFixed Removed"); } // ******************************************** void ParameterHandlerBase::SetFlatPrior(const int i, const bool eL) { // ******************************************** - if (i > params.flatprior.size()) { - MACH3LOG_INFO( - "Can't {} for Cov={}/Param={} because size of Covariance = {}", - __func__, GetName(), i, params.flatprior.size()); - MACH3LOG_ERROR("Fix this in your config file please!"); - throw MaCh3Exception(__FILE__, __LINE__); + + int propidx = parlist.SystematicParameterIndexToPCIndex(i); + if (propidx != ParameterList::ParameterInPCABlock) { + parlist.params.flatprior[i] = eL; } else { - if (eL) { - MACH3LOG_INFO("Setting {} (parameter {}) to flat prior", GetParName(i), - i); - } else { - // HW :: This is useful - MACH3LOG_INFO("Setting {} (parameter {}) to non-flat prior", - GetParName(i), i); - } - params.flatprior[i] = eL; + MACH3LOG_ERROR("You are trying set a flat prior for parameter {}, {}, " + "which has been PCA'd.", + i, GetParName(i)); + throw MaCh3Exception(__FILE__, __LINE__); } } +void ParameterHandlerBase::SetIndivStepScale(const int ParameterIndex, + const double StepScale) { + int idx = parlist.SystematicParameterIndexToPCIndex(ParameterIndex); + if (idx == ParameterList::ParameterInPCABlock) { + MACH3LOG_ERROR("You are trying to set the step scale of parameter {}, {}, " + "which has been PCA'd.", + ParameterIndex, GetParName(ParameterIndex)); + throw MaCh3Exception(__FILE__, __LINE__); + } + + proposer.params.scale[idx] = StepScale; +} + +double ParameterHandlerBase::GetIndivStepScale(const int ParameterIndex) const { + int idx = parlist.SystematicParameterIndexToPCIndex(ParameterIndex); + if (idx == ParameterList::ParameterInPCABlock) { + MACH3LOG_ERROR("You are trying to get the step scale of parameter {}, {}, " + "which has been PCA'd.", + ParameterIndex, GetParName(ParameterIndex)); + throw MaCh3Exception(__FILE__, __LINE__); + } + + return proposer.params.scale[idx]; +} + // ******************************************** void ParameterHandlerBase::SetIndivStepScale( const std::vector &stepscale) { // ******************************************** - if (int(steps.scale.size()) != steps.scale.size()) { + if (int(stepscale.size()) != parlist.NumSystematicBasisParameters()) { MACH3LOG_WARN( "Stepscale vector not equal to number of parameters. Quitting.."); - MACH3LOG_WARN("Size of argument vector: {}", steps.scale.size()); - MACH3LOG_WARN("Expected size: {}", steps.scale.size()); + MACH3LOG_WARN("Size of argument vector: {}", stepscale.size()); + MACH3LOG_WARN("Expected size: {}", parlist.NumSystematicBasisParameters()); return; } - steps.scale = M3::StdVectorToEigen(stepscale); + for (int i = 0; i < parlist.NumSystematicBasisParameters(); ++i) { + + int idx = parlist.SystematicParameterIndexToPCIndex(i); + if (idx == ParameterList::ParameterInPCABlock) { + continue; + } + + proposer.params.scale[idx] = stepscale[i]; + } + PrintIndivStepScale(); } @@ -772,11 +577,16 @@ void ParameterHandlerBase::SetIndivStepScale( void ParameterHandlerBase::PrintIndivStepScale() const { // ******************************************** MACH3LOG_INFO("============================================================"); - MACH3LOG_INFO("{:<{}} | {:<11}", "Parameter:", settings.PrintLength, - "Step scale:"); - for (int iParam = 0; iParam < int(params.fancy_name.size()); iParam++) { - MACH3LOG_INFO("{:<{}} | {:<11}", params.fancy_name[iParam].c_str(), - settings.PrintLength, steps.scale[iParam]); + MACH3LOG_INFO("{:<{}} | {:<11}", "Parameter:", PrintLength, "Step scale:"); + for (int iParam = 0; iParam < parlist.NumSystematicBasisParameters(); + iParam++) { + int idx = parlist.SystematicParameterIndexToPCIndex(iParam); + if (idx == ParameterList::ParameterInPCABlock) { + MACH3LOG_INFO("{:<{}} | {:<11}", GetParName(idx), PrintLength, "PCA'd"); + } else { + MACH3LOG_INFO("{:<{}} | {:<11}", GetParName(idx), PrintLength, + proposer.params.scale[idx]); + } } MACH3LOG_INFO("============================================================"); } @@ -784,8 +594,8 @@ void ParameterHandlerBase::PrintIndivStepScale() const { // ******************************************** void ParameterHandlerBase::ResetIndivStepScale() { // ******************************************** - steps.global_scale = 1.0; - SetIndivStepScale(std::vector(steps.scale.size(), 1.0)); + proposer.params.global_scale = 1; + proposer.params.scale = Eigen::ArrayXd::Ones(proposer.NumParameters()); } // ******************************************** @@ -794,110 +604,63 @@ void ParameterHandlerBase::ResetIndivStepScale() { void ParameterHandlerBase::SetThrowMatrix(TMatrixDSym *cov) { // ******************************************** - if (cov == nullptr) { - MACH3LOG_ERROR("Could not find covariance matrix you provided to {}", - __func__); - throw MaCh3Exception(__FILE__, __LINE__); - } + M3::MakeMatrixPosDef(cov); + Eigen::MatrixXd covariance = M3::ROOTToEigen(*cov); - if (steps.l_proposal.rows() != cov->GetNrows()) { - MACH3LOG_ERROR("Matrix given for throw Matrix is not the same size as the " - "proposal matrix stored in object!"); - MACH3LOG_ERROR("Stored proposal matrix size: {}", steps.l_proposal.rows()); - MACH3LOG_ERROR("Given matrix size: {}", cov->GetNrows()); - throw MaCh3Exception(__FILE__, __LINE__); - } + if (parlist.pca.enabled) { + Eigen::MatrixXd covariance_pc = Eigen::MatrixXd::Zero( + proposer.NumParameters(), proposer.NumParameters()); - if (settings.use_adaptive && AdaptiveHandler.AdaptionUpdate()) { - M3::MakeMatrixClosestPosDef(cov); - Eigen::LLT lltproposal(M3::ROOTToEigen(*cov)); - if (lltproposal.info() != Eigen::Success) { - MACH3LOG_ERROR("Failed to LLT decompose proposal matrix"); - throw MaCh3Exception(__FILE__, __LINE__); - } - steps.l_proposal = lltproposal.matrixL(); + covariance_pc.topLeftCorner(parlist.pca.first_index, + parlist.pca.first_index) = + parlist.params.covariance.topLeftCorner(parlist.pca.first_index, + parlist.pca.first_index); + + covariance_pc.block(parlist.pca.first_index, parlist.pca.first_index, + parlist.pca.npc_parameters(), + parlist.pca.npc_parameters()) = + Eigen::MatrixXd::Identity(parlist.pca.npc_parameters(), + parlist.pca.npc_parameters()); + + covariance_pc.bottomRightCorner(parlist.pca.ntail, parlist.pca.ntail) = + parlist.params.covariance.bottomRightCorner(parlist.pca.ntail, + parlist.pca.ntail); + + proposer.SetProposalMatrix(covariance_pc); } else { - Eigen::LLT lltproposal( - M3::MakeMatrixPosDef(M3::ROOTToEigen(*cov))); - if (lltproposal.info() != Eigen::Success) { - MACH3LOG_ERROR("Failed to LLT decompose proposal matrix"); - throw MaCh3Exception(__FILE__, __LINE__); - } - steps.l_proposal = lltproposal.matrixL(); + proposer.SetProposalMatrix(covariance); } } -// ******************************************** -void ParameterHandlerBase::UpdateThrowMatrix(TMatrixDSym *cov) { - // ******************************************** - SetThrowMatrix(cov); +void ParameterHandlerBase::UpdateThrowMatrix(TMatrixDSym *) { + throw std::runtime_error("ParameterHandlerBase::UpdateThrowMatrix Removed"); +} + +double ParameterHandlerBase::GetThrowMatrix(const int, const int) const { + throw std::runtime_error("ParameterHandlerBase::GetThrowMatrix(i,j) Removed"); +} + +TH2D *ParameterHandlerBase::GetCorrelationMatrix() { + return M3::GetCorrelationMatrix(GetName(), parlist.params.name, + parlist.params.covariance); +} + +const double *ParameterHandlerBase::RetPointer(const int) { + throw std::runtime_error("ParameterHandlerBase::RetPointer Removed"); } // ******************************************** // HW : Here be adaption -void ParameterHandlerBase::InitialiseAdaption(const YAML::Node &adapt_manager) { +void ParameterHandlerBase::InitialiseAdaption(const YAML::Node &) { // ******************************************** - if (pca.enabled) { - MACH3LOG_ERROR("PCA has been enabled and now trying to enable Adaption. " - "Right now both configuration don't work with each other"); - throw MaCh3Exception(__FILE__, __LINE__); - } - if (settings.use_adaptive) { - MACH3LOG_ERROR("Adaptive Handler has already been initialise can't do it " - "again so skipping."); - return; - } - - // Now we read the general settings [these SHOULD be common across all - // matrices!] - // bool success = AdaptiveHandler.InitFromConfig(adapt_manager, settings.name, - // &steps.current, - // ¶ms.error); - // if (!success) { - // return; - // } - - AdaptiveHandler.Print(); - - // Next let"s check for external matrices - // We"re going to grab this info from the YAML manager - if (!GetFromManager(adapt_manager["AdaptionOptions"]["Covariance"] - [settings.name]["UseExternalMatrix"], - false, __FILE__, __LINE__)) { - MACH3LOG_WARN( - "Not using external matrix for {}, initialising adaption from scratch", - settings.name); - // If we don't have a covariance matrix to start from for adaptive tune we - // need to make one! - settings.use_adaptive = true; - AdaptiveHandler.CheckMatrixValidityForAdaption(GetCovMatrix()); - AdaptiveHandler.CreateNewAdaptiveCovariance(); - return; - } + throw std::runtime_error( + "ParameterHandlerBase::InitialiseAdaption Not Yet Implemented"); +} - // Finally, we accept that we want to read the matrix from a file! - auto external_file_name = GetFromManager( - adapt_manager["AdaptionOptions"]["Covariance"][settings.name] - ["ExternalMatrixFileName"], - "", __FILE__, __LINE__); - auto external_matrix_name = GetFromManager( - adapt_manager["AdaptionOptions"]["Covariance"][settings.name] - ["Externalsettings.name"], - "", __FILE__, __LINE__); - auto external_mean_name = GetFromManager( - adapt_manager["AdaptionOptions"]["Covariance"][settings.name] - ["ExternalMeansName"], - "", __FILE__, __LINE__); - - AdaptiveHandler.SetThrowMatrixFromFile( - external_file_name, external_matrix_name, external_mean_name, - settings.use_adaptive); - SetThrowMatrix(AdaptiveHandler.GetAdaptiveCovariance()); - - ResetIndivStepScale(); - - MACH3LOG_INFO("Successfully Set External Throw Matrix Stored in {}", - external_file_name); +void ParameterHandlerBase::SaveAdaptiveToFile(const std::string &, + const std::string &) { + throw std::runtime_error( + "ParameterHandlerBase::SaveAdaptiveToFile Not Yet Implemented"); } // ******************************************** @@ -905,71 +668,22 @@ void ParameterHandlerBase::InitialiseAdaption(const YAML::Node &adapt_manager) { void ParameterHandlerBase::UpdateAdaptiveCovariance() { // ******************************************** // Updates adaptive matrix - // First we update the total means - - // Skip this if we're at a large number of steps - if (AdaptiveHandler.SkipAdaption()) { - AdaptiveHandler.IncrementNSteps(); - return; - } - - /// Need to adjust the scale every step - if (AdaptiveHandler.GetUseRobbinsMonro()) { - bool verbose = false; -#ifdef DEBUG - verbose = true; -#endif - AdaptiveHandler.UpdateRobbinsMonroScale(); - SetStepScale(AdaptiveHandler.GetAdaptionScale(), verbose); - } - - // Call main adaption function - AdaptiveHandler.UpdateAdaptiveCovariance(); - - // Set scales to 1 * optimal scale - if (AdaptiveHandler.IndivStepScaleAdapt()) { - ResetIndivStepScale(); - SetStepScale(AdaptiveHandler.GetAdaptionScale()); - } - - if (AdaptiveHandler.UpdateMatrixAdapt()) { - UpdateThrowMatrix( - AdaptiveHandler.GetAdaptiveCovariance()); // Now we update and continue! - // Also Save the adaptive to file - AdaptiveHandler.SaveAdaptiveToFile(AdaptiveHandler.GetOutFileName(), - GetName()); - } - - AdaptiveHandler.IncrementNSteps(); + throw std::runtime_error( + "ParameterHandlerBase::UpdateAdaptiveCovariance Not Yet Implemented"); } // ******************************************** // KS: After step scale, prefit etc. value were modified save this modified // config. -void ParameterHandlerBase::SaveUpdatedMatrixConfig( - std::string const &filename) { +void ParameterHandlerBase::SaveUpdatedMatrixConfig() { // ******************************************** - if (!config.YAMLDoc) { - MACH3LOG_CRITICAL("Yaml node hasn't been initialised for matrix {}, " - "something is not right", - settings.name); - MACH3LOG_CRITICAL("I am not throwing error but should be investigated"); - return; - } - - YAML::Node copyNode = config.YAMLDoc; - int i = 0; + throw std::runtime_error( + "ParameterHandlerBase::SaveUpdatedMatrixConfig Not Yet Implemented"); +} - for (YAML::Node param : copyNode["Systematics"]) { - // KS: Feel free to update it, if you need updated prefit value etc - param["Systematic"]["StepScale"]["MCMC"] = - MaCh3Utils::FormatDouble(steps.scale[i], 4); - i++; - } - // Save the modified node to a file - std::ofstream fout(filename); - fout << copyNode; - fout.close(); +void ParameterHandlerBase::SetNumberOfSteps(const int) { + throw std::runtime_error( + "ParameterHandlerBase::SetNumberOfSteps Not Yet Implemented"); } // ******************************************** @@ -977,9 +691,8 @@ bool ParameterHandlerBase::AppliesToSample( const int SystIndex, const std::string &SampleName) const { // ******************************************** // Empty means apply to all - if (!params.samples[SystIndex].size()) { + if (parlist.params.samples[SystIndex].size() == 0) return true; - } // Make a copy and to lower case to not be case sensitive std::string SampleNameCopy = SampleName; @@ -995,9 +708,9 @@ bool ParameterHandlerBase::AppliesToSample( bool Applies = false; - for (size_t i = 0; i < params.samples[SystIndex].size(); i++) { + for (size_t i = 0; i < parlist.params.samples[SystIndex].size(); i++) { // Convert to low case to not be case sensitive - std::string pattern = params.samples[SystIndex][i]; + std::string pattern = parlist.params.samples[SystIndex][i]; std::transform(pattern.begin(), pattern.end(), pattern.begin(), ::tolower); // Replace '*' in the pattern with '.*' for regex matching @@ -1019,9 +732,10 @@ bool ParameterHandlerBase::AppliesToSample( // ******************************************** // Set proposed parameter values vector to be base on tune values -void ParameterHandlerBase::SetTune(const std::string &TuneName) { +void ParameterHandlerBase::SetTune(const std::string &) { // ******************************************** - SetParameters(Tunes.GetTune(TuneName)); + throw std::runtime_error( + "ParameterHandlerBase::SaveUpdatedMatrixConfig Not Yet Implemented"); } // ************************************* diff --git a/Parameters/ParameterHandlerBase.h b/Parameters/ParameterHandlerBase.h index f6c0a1e23..5ccbb2100 100644 --- a/Parameters/ParameterHandlerBase.h +++ b/Parameters/ParameterHandlerBase.h @@ -2,17 +2,10 @@ // MaCh3 includes #include "Manager/Manager.h" -#include "Parameters/AdaptiveMCMCHandler.h" -#include "Parameters/PCAHandler.h" -#include "Parameters/ParameterHandlerUtils.h" -#include "Parameters/ParameterTunes.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Walloca" -#include "Eigen/Dense" -#pragma GCC diagnostic pop -#include +#include "Parameters/ParameterHandlerUtils.h" +#include "Parameters/ParameterList.h" +#include "Parameters/StepProposer.h" /// @brief Base class responsible for handling of systematic error parameters. /// Capable of using PCA or using adaptive throw matrix @@ -23,26 +16,6 @@ /// @author Kamil Skwarczynski class ParameterHandlerBase { public: - struct ParamInfo { - std::string name, fancy_name; - double prefit, error, stepscale; - std::array bounds; - bool flatprior, isfree; - std::vector affected_samples; - }; - - void AddParameters(std::vector const ¶ms); - void AddParameter(ParamInfo const ¶m) { - AddParameters({ - param, - }); - } - - void SetParameterCorrelation(int pidi, int pidj, double corr); - - void SetParameterAllCorrelations( - int paramid, std::map const &correlations); - /// @brief ETA - constructor for a YAML file /// @param YAMLFile A vector of strings representing the YAML files used for /// initialisation of matrix @@ -50,29 +23,18 @@ class ParameterHandlerBase { /// @param threshold PCA threshold from 0 to 1. Default is -1 and means no PCA /// @param FirstPCAdpar First PCA parameter that will be decomposed. /// @param LastPCAdpar First PCA parameter that will be decomposed. - static ParameterHandlerBase - MakeFromYAML(const std::vector &YAMLFiles, std::string name, - double threshold = -1, int FirstPCAdpar = -999, - int LastPCAdpar = -999) { - (void)threshold; - (void)FirstPCAdpar; - (void)LastPCAdpar; - return ParameterHandlerBase(YAMLFiles, name); - } + ParameterHandlerBase(const std::vector &YAMLFile, + std::string name, double threshold = -1, + int FirstPCAdpar = -999, int LastPCAdpar = -999); /// @brief "Usual" constructors from root file /// @param name Matrix name /// @param file Path to matrix root file - static ParameterHandlerBase - MakeFromTMatrix(std::string name, std::string file, double threshold = -1, - int FirstPCAdpar = -999, int LastPCAdpar = -999) { - (void)threshold; - (void)FirstPCAdpar; - (void)LastPCAdpar; - return ParameterHandlerBase(name, file); - } + ParameterHandlerBase(std::string name, std::string file, + double threshold = -1, int FirstPCAdpar = -999, + int LastPCAdpar = -999); /// @brief Destructor - virtual ~ParameterHandlerBase() {}; + virtual ~ParameterHandlerBase() {} /// @defgroup ParameterHandlerSetters Parameter Handler Setters /// Group of functions to set various parameters, names, and values. @@ -89,13 +51,14 @@ class ParameterHandlerBase { void SetCovMatrix(TMatrixDSym *cov); /// @brief Set matrix name /// @ingroup ParameterHandlerSetters - void SetName(const std::string &name) { settings.name = name; } + std::string matrixName; + void SetName(const std::string &name) { matrixName = name; } /// @brief change parameter name /// @param i Parameter index /// @param name new name which will be set /// @ingroup ParameterHandlerSetters void SetParName(const int i, const std::string &name) { - params.name.at(i) = name; + parlist.params.name.at(i) = name; } /// @brief Set value of single param to a given value /// @ingroup ParameterHandlerSetters @@ -114,7 +77,7 @@ class ParameterHandlerBase { /// @param i Parameter index /// @param val new value which will be set /// @ingroup ParameterHandlerSetters - void SetParProp(const int i, const double val) { steps.proposed[i] = val; } + void SetParProp(const int i, const double val); /// @brief Set parameter values using vector, it has to have same size as /// covariance class /// @param pars Vector holding new values for every parameter @@ -130,13 +93,11 @@ class ParameterHandlerBase { /// @param i Parameter index /// @param rand New value for random number /// @ingroup ParameterHandlerSetters - void SetRandomThrow(const int i, const double rand) { - throws.random_vector[i] = rand; - } + void SetRandomThrow(const int i, const double rand); /// @brief Get random value useful for debugging/CI /// @param i Parameter index /// @ingroup ParameterHandlerGetters - double GetRandomThrow(const int i) const { return throws.random_vector[i]; } + double GetRandomThrow(const int i) const; /// @brief set branches for output file /// @param tree Tree to which we will save branches @@ -157,9 +118,7 @@ class ParameterHandlerBase { /// @param ParameterIndex Parameter Index /// @param StepScale Value of individual step scale /// @ingroup ParameterHandlerSetters - void SetIndivStepScale(const int ParameterIndex, const double StepScale) { - steps.scale[ParameterIndex] = StepScale; - } + void SetIndivStepScale(const int ParameterIndex, const double StepScale); /// @brief DB Function to set fIndivStepScale from a vector (Can be used from /// execs and inside covariance constructors) /// @param stepscale Vector of individual step scale, should have same @@ -167,14 +126,12 @@ class ParameterHandlerBase { void SetIndivStepScale(const std::vector &stepscale); /// @brief KS: In case someone really want to change this /// @ingroup ParameterHandlerSetters - inline void SetPrintLength(const unsigned int PriLen) { - settings.PrintLength = PriLen; - } + int PrintLength = 35; + void SetPrintLength(const unsigned int PriLen) { PrintLength = PriLen; } /// @brief KS: After step scale, prefit etc. value were modified save this /// modified config. - void - SaveUpdatedMatrixConfig(std::string const &filename = "Modified_Matrix.yaml"); + void SaveUpdatedMatrixConfig(); /// @brief Throw the proposed parameter by mag sigma. Should really just have /// the user specify this throw by having argument double @@ -208,7 +165,7 @@ class ParameterHandlerBase { /// @note /// - If `_fFlatPrior[i]` is `true`, the parameter is excluded from the /// calculation. - double CalcLikelihood() const _noexcept_; + double CalcLikelihood() _noexcept_; /// @brief Return CalcLikelihood if some params were thrown out of boundary /// return _LARGE_LOGL_ /// @ingroup ParameterHandlerGetters @@ -216,35 +173,35 @@ class ParameterHandlerBase { /// @brief Return covariance matrix /// @ingroup ParameterHandlerGetters - TMatrixDSym const *GetCovMatrix() const { return &root_copies.covariance; } + TMatrixDSym *covMatrix; + TMatrixDSym *GetCovMatrix() const { return covMatrix; } /// @brief Return inverted covariance matrix /// @ingroup ParameterHandlerGetters - TMatrixDSym const *GetInvCovMatrix() const { - return &root_copies.inv_covariance; - } + TMatrixDSym *invCovMatrix; + TMatrixDSym *GetInvCovMatrix() const { return invCovMatrix; } /// @brief Return inverted covariance matrix /// @ingroup ParameterHandlerGetters - double GetInvCovMatrix(const int i, const int j) const { - return params.inv_covariance(i, j); - } + double GetInvCovMatrix(const int i, const int j) const; /// @brief Return correlated throws /// @param i Parameter index /// @ingroup ParameterHandlerGetters - double GetCorrThrows(const int i) const { return throws.values[i]; } + double GetCorrThrows(const int i) const; /// @brief Get if param has flat prior or not /// @param i Parameter index /// @ingroup ParameterHandlerGetters - bool GetFlatPrior(const int i) const { return params.flatprior[i]; } + bool GetFlatPrior(const int i) const; /// @brief Get name of covariance /// @ingroup ParameterHandlerGetters - std::string GetName() const { return settings.name; } + std::string GetName() const { return matrixName; } /// @brief Get name of parameter /// @param i Parameter index /// @ingroup ParameterHandlerGetters - std::string GetParName(const int i) const { return params.name[i]; } + std::string GetParName(const int i) const { + return parlist.params.name.at(i); + } /// @brief Get index based on name /// @ingroup ParameterHandlerGetters @@ -254,22 +211,24 @@ class ParameterHandlerBase { /// @param i Parameter index /// @ingroup ParameterHandlerGetters std::string GetParFancyName(const int i) const { - return params.fancy_name[i]; + return parlist.params.fancy_name.at(i); + ; } /// @brief Get name of input file /// @ingroup ParameterHandlerGetters - std::string GetInputFile() const { return config.inputFiles[0]; } + std::string inputFile; + std::string GetInputFile() const { return inputFile; } /// @brief Get diagonal error for ith parameter /// @param i Parameter index /// @ingroup ParameterHandlerGetters double GetDiagonalError(const int i) const { - return std::sqrt(params.covariance(i, i)); + return std::sqrt(parlist.params.covariance(i, i)); } /// @brief Get the error for the ith parameter /// @param i Parameter index /// @ingroup ParameterHandlerGetters - double GetError(const int i) const { return params.error[i]; } + double GetError(const int i) const { return parlist.params.error[i]; } /// @brief Adaptive Step Tuning Stuff void ResetIndivStepScale(); @@ -279,13 +238,11 @@ class ParameterHandlerBase { void InitialiseAdaption(const YAML::Node &adapt_manager); /// @brief Save adaptive throw matrix to file void SaveAdaptiveToFile(const std::string &outFileName, - const std::string &systematicName) { - AdaptiveHandler.SaveAdaptiveToFile(outFileName, systematicName); - } + const std::string &systematicName); /// @brief Do we adapt or not /// @ingroup ParameterHandlerGetters - bool GetDoAdaption() const { return settings.use_adaptive; } + bool GetDoAdaption() const { return false; } /// @brief Use new throw matrix, used in adaptive MCMC /// @ingroup ParameterHandlerSetters void SetThrowMatrix(TMatrixDSym *cov); @@ -294,21 +251,15 @@ class ParameterHandlerBase { /// @brief Set number of MCMC step, when running adaptive MCMC it is updated /// with given frequency. We need number of steps to determine frequency. /// @ingroup ParameterHandlerSetters - void SetNumberOfSteps(const int nsteps) { - AdaptiveHandler.SetTotalSteps(nsteps); - if (AdaptiveHandler.AdaptionUpdate()) { - ResetIndivStepScale(); - } - } + void SetNumberOfSteps(const int nsteps); /// @brief Get matrix used for step proposal /// @ingroup ParameterHandlerGetters - TMatrixDSym const *GetThrowMatrix() const { return &root_copies.proposal; } + TMatrixDSym *throwMatrix; + TMatrixDSym *GetThrowMatrix() const { return throwMatrix; } /// @brief Get matrix used for step proposal /// @ingroup ParameterHandlerGetters - double GetThrowMatrix(const int i, const int j) const { - return steps.l_proposal(i, j); - } + double GetThrowMatrix(const int i, const int j) const; /// @brief KS: Convert covariance matrix to correlation matrix and return TH2D /// which can be used for fancy plotting @@ -320,52 +271,77 @@ class ParameterHandlerBase { /// @ingroup ParameterHandlerGetters TH2D *GetCorrelationMatrix(); + /// @brief DB Pointer return to param position + /// + /// @param iParam The index of the parameter in the vector. + /// @return A pointer to the parameter value at the specified index. + /// + /// @warning ETA - This might be a bit squiffy? If the vector gots moved from + /// say a push_back then the pointer is no longer valid... maybe need a better + /// way to deal with this? It was fine before when the return was to an + /// element of a new array. There must be a clever C++ way to be careful + const double *RetPointer(const int iParam); + + /// @brief Get a reference to the proposed parameter values + /// Can be useful if you want to track these without having to copy values + /// using getProposed() + mutable std::vector _fPropVal; + const std::vector &GetParPropVec() const { + _fPropVal = M3::EigenToStdVector(proposed); + return _fPropVal; + } + /// @brief Get total number of parameters /// @ingroup ParameterHandlerGetters - int GetNumParams() const { return int(params.prefit.size()); } + int GetNumParams() const { return parlist.NumSystematicBasisParameters(); } /// @brief Get the pre-fit values of the parameters. /// @ingroup ParameterHandlerGetters std::vector GetPreFitValues() const { - return M3::EigenToStdVector(params.prefit); + return M3::EigenToStdVector(parlist.params.prefit); } /// @brief Get vector of all proposed parameter values /// @ingroup ParameterHandlerGetters - std::vector GetProposed() const { - return M3::EigenToStdVector(steps.proposed); - } + std::vector GetProposed() const; /// @brief Get proposed parameter value /// @param i Parameter index /// @ingroup ParameterHandlerGetters - double GetParProp(const int i) const { return steps.proposed[i]; } + double GetParProp(const int i) const { return proposed[i]; } /// @brief Get current parameter value /// @param i Parameter index /// @ingroup ParameterHandlerGetters - double GetParCurr(const int i) const { return steps.current[i]; } + double GetParCurr(const int i) const { return current[i]; } + /// @brief Get vector of current parameter values + /// @ingroup ParameterHandlerGetters + mutable std::vector _fCurrVal; + const std::vector &GetParCurrVec() const { + _fCurrVal = M3::EigenToStdVector(current); + return _fCurrVal; + } /// @brief Get prior parameter value /// @param i Parameter index /// @ingroup ParameterHandlerGetters - double GetParInit(const int i) const { return params.prefit[i]; } + double GetParInit(const int i) const { return parlist.params.prefit[i]; } /// @brief Get upper parameter bound in which it is physically valid /// @param i Parameter index /// @ingroup ParameterHandlerGetters - double GetUpperBound(const int i) const { return params.upbound[i]; } + double GetUpperBound(const int i) const { return parlist.params.upbound[i]; } /// @brief Get lower parameter bound in which it is physically valid /// @param i Parameter index /// @ingroup ParameterHandlerGetters - double GetLowerBound(const int i) const { return params.lowbound[i]; } + double GetLowerBound(const int i) const { return parlist.params.lowbound[i]; } /// @brief Get individual step scale for selected parameter /// @param ParameterIndex Parameter index /// @ingroup ParameterHandlerGetters - double GetIndivStepScale(const int i) const { return steps.scale[i]; } + double GetIndivStepScale(const int ParameterIndex) const; /// @brief Get global step scale for covariance object /// @ingroup ParameterHandlerGetters - double GetGlobalStepScale() const { return steps.global_scale; } + double GetGlobalStepScale() const { return proposer.params.global_scale; } /// @brief Get number of params which will be different depending if using /// Eigen decomposition or not /// @ingroup ParameterHandlerGetters - int GetNParameters() const { return GetNumParams(); } + int GetNParameters() const { return parlist.NumSystematicBasisParameters(); } /// @brief Print prior value for every parameter void PrintNominal() const; @@ -420,8 +396,7 @@ class ParameterHandlerBase { void ToggleFixParameter(const std::string &name); /// @brief Is parameter fixed or not /// @param i Parameter index - bool IsParameterFixed(const int i) const { return !params.isfree[i]; } - + bool IsParameterFixed(const int i) const; /// @brief Is parameter fixed or not /// @param name Name of parameter you want to check if is fixed bool IsParameterFixed(const std::string &name) const; @@ -438,21 +413,12 @@ class ParameterHandlerBase { int LastPCAdpar); /// @brief is PCA, can use to query e.g. LLH scans - bool IsPCA() const { return pca.enabled; } + bool IsPCA() const { return parlist.pca.enabled; } /// @brief Getter to return a copy of the YAML node /// @ingroup ParameterHandlerGetters - YAML::Node GetConfig() const { return config.YAMLDoc; } - - /// @brief Get pointer for AdaptiveHandler - /// @ingroup ParameterHandlerGetters - adaptive_mcmc::AdaptiveMCMCHandler const &GetAdaptiveHandler() const { - if (!settings.use_adaptive) { - MACH3LOG_ERROR("Am not running in Adaptive mode"); - throw MaCh3Exception(__FILE__, __LINE__); - } - return AdaptiveHandler; - } + YAML::Node _fYAMLDoc; + YAML::Node GetConfig() { return _fYAMLDoc; } /// @brief KS: Set proposed parameter values vector to be base on tune values, /// for example set proposed values to be of generated or maybe PostND @@ -473,125 +439,15 @@ class ParameterHandlerBase { std::vector &BranchNames); protected: - ParameterHandlerBase(); - - /// @brief "Usual" constructors from root file - /// @param name Matrix name - /// @param file Path to matrix root file - ParameterHandlerBase(std::string const &name, std::string const &file); - - /// @brief ETA - constructor for a YAML file - /// @param YAMLFile A vector of strings representing the YAML files used for - /// initialisation of matrix - /// @param name Matrix name - /// @param threshold PCA threshold from 0 to 1. Default is -1 and means no PCA - /// @param FirstPCAdpar First PCA parameter that will be decomposed. - /// @param LastPCAdpar First PCA parameter that will be decomposed. - ParameterHandlerBase(const std::vector &YAMLFiles, - std::string const &name); - - /// @brief sets throw matrix from a file - /// @param matrix_file_name name of file matrix lives in - /// @param matrix_name name of matrix in file - /// @param means_name name of means vec in file - void SetThrowMatrixFromFile(const std::string &matrix_file_name, - const std::string &matrix_name, - const std::string &means_name); - /// @brief Check if parameter is affecting given sample name /// @param SystIndex number of parameter /// @param SampleName The Sample name used to filter parameters. bool AppliesToSample(const int SystIndex, const std::string &SampleName) const; - /// @brief KS: Flip parameter around given value, for example mass ordering - /// around 0 - /// @param index parameter index you want to flip - /// @param FlipPoint Value around which flipping is done - void FlipParameterValue(const int index, const double FlipPoint); - - /// @brief HW :: This method is a tad hacky but modular arithmetic gives me a - /// headache. - /// @author Henry Wallace - void CircularParBounds(const int i, const double LowBound, - const double UpBound); - - /// @brief Enable special proposal - void EnableSpecialProposal(const YAML::Node ¶m, const int Index); - - /// @brief Perform Special Step Proposal - /// @warning KS: Following Asher comment we do "Step->Circular Bounds->Flip" - void SpecialStepProposal(); - - struct { - bool enabled; - std::array block_indices; - Eigen::MatrixXd ortho_to_physics; - } pca; - - struct { - Eigen::VectorXd random_vector, values; - } throws; - - struct { - Eigen::VectorXd current, proposed, scale; - double global_scale; - Eigen::MatrixXd l_proposal; - } steps; - - struct { - bool enabled; - - // param_index, flip pivot value - std::vector> flips; - // param_index, low bound, up bound - std::vector> circ_bounds; - - } special_proposal; - - struct { - std::vector name, fancy_name; - Eigen::VectorXd prefit, error, lowbound, upbound; - Eigen::VectorXi flatprior, isfree; - std::vector> samples; - - Eigen::MatrixXd covariance; - Eigen::MatrixXd inv_covariance; - } params; - - struct { - std::ranlux48 e1; - std::normal_distribution gaus; - std::uniform_real_distribution unif; - } rng; - - // to retain external interface where possible - struct { - TMatrixDSym covariance; - TMatrixDSym inv_covariance; - TMatrixDSym proposal; - } root_copies; - - struct { - std::string name; - - /// Are we using AMCMC? - bool use_adaptive; - - /// KS: This is used when printing parameters, sometimes we have super long - /// parameters name, we want to flexibly adjust couts - int PrintLength; - } settings; - - struct { - /// The input root file we read in - std::vector inputFiles; - /// Stores config describing systematics - YAML::Node YAMLDoc; - } config; - - /// Struct containing information about adaption - adaptive_mcmc::AdaptiveMCMCHandler AdaptiveHandler; - /// Struct containing information about adaption - ParameterTunes Tunes; + ParameterList parlist; + StepProposer proposer; + + mutable Eigen::ArrayXd current; + mutable Eigen::ArrayXd proposed; }; diff --git a/Parameters/ParameterList.cpp b/Parameters/ParameterList.cpp index c6863be03..4df0421f3 100644 --- a/Parameters/ParameterList.cpp +++ b/Parameters/ParameterList.cpp @@ -33,6 +33,10 @@ void ParameterList::AddParameters(std::vector const &new_params) { params.stepscale[new_block_start + i] = new_params[i].stepscale; params.flatprior[new_block_start + i] = new_params[i].flatprior; params.isfree[new_block_start + i] = new_params[i].isfree; + + params.covariance.row(new_block_start + i).setZero(); + params.covariance.col(new_block_start + i).setZero(); + params.covariance(new_block_start + i, new_block_start + i) = new_params[i].error * new_params[i].error; } @@ -60,10 +64,8 @@ void ParameterList::SetParameterCorrelation(int pidi, int pidj, double corr) { void ParameterList::SetParameterAllCorrelations( int paramid, std::map const &correlations) { - params.covariance.row(paramid) = - Eigen::VectorXd::Zero(params.covariance.rows()); - params.covariance.col(paramid) = - Eigen::VectorXd::Zero(params.covariance.rows()); + params.covariance.row(paramid).setZero(); + params.covariance.col(paramid).setZero(); params.covariance(paramid, paramid) = params.error[paramid] * params.error[paramid]; @@ -73,7 +75,7 @@ void ParameterList::SetParameterAllCorrelations( } } -int ParameterList::FindParameter(std::string const &name) { +int ParameterList::FindParameter(std::string const &name) const { auto pit = std::find(params.name.begin(), params.name.end(), name); if (pit == params.name.end()) { MACH3LOG_ERROR("ParameterList manages no parameter named {}.", name); @@ -94,10 +96,11 @@ void ParameterList::ConstructTruncatedPCA(double threshold, int first, int blocksize = (last + 1) - first; - pca.pc_to_syst_rotation = CalculateTruncatedPCARotation( - params.covariance.block(pca.first_index, pca.first_index, blocksize, - blocksize), - threshold); + std::tie(pca.pc_to_syst_rotation, pca.syst_to_pc_rotation) = + CalculateTruncatedPCARotation( + params.covariance.block(pca.first_index, pca.first_index, blocksize, + blocksize), + threshold); MACH3LOG_INFO("ParameterList::ConstructTruncatedPCA: threshold = {}, " "nsystparams = {} truncated to {} PC params.", @@ -106,7 +109,7 @@ void ParameterList::ConstructTruncatedPCA(double threshold, int first, } void ParameterList::RotatePCParameterValuesToSystematicBasis( - Eigen::ArrayXd const &pc_vals, Eigen::ArrayXd &systematic_vals) { + Eigen::ArrayXd const &pc_vals, Eigen::ArrayXd &systematic_vals) const { systematic_vals.head(pca.first_index) = pc_vals.head(pca.first_index); @@ -114,17 +117,20 @@ void ParameterList::RotatePCParameterValuesToSystematicBasis( // systematic basis into the by using the PCA matrix. // 2) add on the prefit values from to these deltas to return to the // systematic basis. + auto syst_param_delta = + (pca.pc_to_syst_rotation * + pc_vals.segment(pca.first_index, pca.npc_parameters()).matrix()) + .array(); + systematic_vals.segment(pca.first_index, pca.nrotated_syst_parameters()) = params.prefit.segment(pca.first_index, pca.nrotated_syst_parameters()) + - (pc_vals.segment(pca.first_index, pca.npc_parameters()).matrix() * - pca.pc_to_syst_rotation) - .array(); + syst_param_delta; systematic_vals.tail(pca.ntail) = pc_vals.tail(pca.ntail); } void ParameterList::RotateSystematicParameterValuesToPCBasis( - Eigen::ArrayXd const &systematic_vals, Eigen::ArrayXd &pc_vals) { + Eigen::ArrayXd const &systematic_vals, Eigen::ArrayXd &pc_vals) const { pc_vals.head(pca.first_index) = systematic_vals.head(pca.first_index); @@ -132,18 +138,18 @@ void ParameterList::RotateSystematicParameterValuesToPCBasis( // PC basis has a CV of 0 // 2) rotate the resulting parameter 'deltas' in the systematic basis into the // PC basis by using the transpose of the PCA matrix + auto syst_param_delta = + (systematic_vals - params.prefit) + .segment(pca.first_index, pca.nrotated_syst_parameters()) + .matrix(); + pc_vals.segment(pca.first_index, pca.npc_parameters()) = - ((systematic_vals.segment(pca.first_index, - pca.nrotated_syst_parameters()) - - params.prefit.segment(pca.first_index, pca.nrotated_syst_parameters())) - .matrix() * - pca.pc_to_syst_rotation.transpose()) - .array(); + (pca.syst_to_pc_rotation * syst_param_delta).array(); pc_vals.tail(pca.ntail) = systematic_vals.tail(pca.ntail); } -int ParameterList::SystematicParameterIndexToPCIndex(int i) { +int ParameterList::SystematicParameterIndexToPCIndex(int i) const { if ((!pca.enabled) || (i < pca.first_index)) { return i; } else if ((i >= pca.first_index) && (i <= pca.last_index)) { @@ -165,7 +171,7 @@ double ParameterList::Chi2(Eigen::ArrayXd const &systematic_vals) { return diff.transpose() * params.inv_covariance * diff; } -StepProposer ParameterList::MakeProposer() { +StepProposer ParameterList::MakeProposer() const { StepProposer proposer; if (!pca.enabled) { @@ -176,8 +182,7 @@ StepProposer ParameterList::MakeProposer() { // the PC parameter basis. // copy the unrotated parameters and covariance blocks - int nproposer_parameters = - pca.first_index + pca.npc_parameters() + pca.ntail; + int nproposer_parameters = NumPCBasisParameters(); Eigen::ArrayXd prefit_pc = Eigen::ArrayXd::Zero(nproposer_parameters); @@ -201,7 +206,11 @@ StepProposer ParameterList::MakeProposer() { proposer.SetProposalMatrix(covariance_pc); } - for (int i = 0; i < params.prefit.size(); ++i) { + // PCA parameters get a step scale of 1 + proposer.params.scale = Eigen::ArrayXd::Ones(NumPCBasisParameters()); + proposer.params.isfree = Eigen::ArrayXi::Ones(NumPCBasisParameters()); + + for (int i = 0; i < NumSystematicBasisParameters(); ++i) { int idx = SystematicParameterIndexToPCIndex(i); @@ -229,12 +238,21 @@ StepProposer ParameterList::MakeProposer() { {idx, std::get<1>(params.circ_bounds[i]), std::get<2>(params.circ_bounds[i])}); } + + if (idx == ParameterInPCABlock) { + continue; + } + + proposer.params.scale[idx] = params.stepscale[i]; + proposer.params.isfree[idx] = params.isfree[i]; } + proposer.params.global_scale = 1; + return proposer; } -std::string ParameterList::SystematicParameterToString(int i) { +std::string ParameterList::SystematicParameterToString(int i) const { return fmt::format( R"(name: {}, fancy_name: {} prefit: {:.3g}, error: {:.3g}, bounds: [ {:.3g}, {:.3g} ], stepscale: {:.3g} @@ -247,7 +265,7 @@ std::string ParameterList::SystematicParameterToString(int i) { params.flip_pivot[i].first, std::get<0>(params.circ_bounds[i])); } -ParameterList MakeFromYAML(YAML::Node const &config) { +ParameterList ParameterList::MakeFromYAML(YAML::Node const &config) { ParameterList parlist; @@ -426,6 +444,11 @@ ParameterList MakeFromYAML(YAML::Node const &config) { parlist.params.covariance = M3::MakeMatrixPosDef(parlist.params.covariance); + parlist.pca.enabled = false; + parlist.pca.first_index = parlist.NumSystematicBasisParameters(); + parlist.pca.last_index = parlist.NumSystematicBasisParameters(); + parlist.pca.ntail = 0; + MACH3LOG_INFO("----------------"); MACH3LOG_INFO("Found {} systematics parameters in total", parlist.params.prefit.size()); @@ -434,7 +457,8 @@ ParameterList MakeFromYAML(YAML::Node const &config) { return parlist; } -ParameterList MakeFromYAML(const std::vector &YAMLFiles) { +ParameterList +ParameterList::MakeFromYAML(const std::vector &YAMLFiles) { MACH3LOG_INFO("Constructing instance of ParameterHandler using: "); for (auto const &yf : YAMLFiles) { @@ -461,8 +485,8 @@ ParameterList MakeFromYAML(const std::vector &YAMLFiles) { return parlist; } -ParameterList MakeFromTMatrix(std::string const &name, - std::string const &file) { +ParameterList ParameterList::MakeFromTMatrix(std::string const &name, + std::string const &file) { ParameterList parlist; @@ -503,5 +527,10 @@ ParameterList MakeFromTMatrix(std::string const &name, parlist.params.covariance = M3::MakeMatrixPosDef(M3::ROOTToEigen(*CovMat)); + parlist.pca.enabled = false; + parlist.pca.first_index = parlist.NumSystematicBasisParameters(); + parlist.pca.last_index = parlist.NumSystematicBasisParameters(); + parlist.pca.ntail = 0; + return parlist; } diff --git a/Parameters/ParameterList.h b/Parameters/ParameterList.h index c9f15a462..e9a427678 100644 --- a/Parameters/ParameterList.h +++ b/Parameters/ParameterList.h @@ -58,47 +58,47 @@ struct ParameterList { }); } - std::string SystematicParameterToString(int i); + std::string SystematicParameterToString(int i) const; void SetParameterCorrelation(int pidi, int pidj, double corr); void SetParameterAllCorrelations( int paramid, std::map const &correlations); - int FindParameter(std::string const &name); - int NumSystematicBasisParameters() { return int(params.prefit.size()); } + int FindParameter(std::string const &name) const; + int NumSystematicBasisParameters() const { return int(params.prefit.size()); } struct { bool enabled; int first_index, last_index, ntail; Eigen::MatrixXd pc_to_syst_rotation; + Eigen::MatrixXd syst_to_pc_rotation; int nrotated_syst_parameters() const { - return int(pc_to_syst_rotation.cols()); + return int(pc_to_syst_rotation.rows()); } - int npc_parameters() const { return int(pc_to_syst_rotation.rows()); } + int npc_parameters() const { return int(pc_to_syst_rotation.cols()); } } pca; void ConstructTruncatedPCA(double threshold, int first, int last); void RotatePCParameterValuesToSystematicBasis( - Eigen::ArrayXd const &pc_vals, Eigen::ArrayXd &systematic_vals); + Eigen::ArrayXd const &pc_vals, Eigen::ArrayXd &systematic_vals) const; void RotateSystematicParameterValuesToPCBasis( - Eigen::ArrayXd const &systematic_vals, Eigen::ArrayXd &pc_vals); + Eigen::ArrayXd const &systematic_vals, Eigen::ArrayXd &pc_vals) const; constexpr static int ParameterInPCABlock = std::numeric_limits::max(); - int SystematicParameterIndexToPCIndex(int i); + int SystematicParameterIndexToPCIndex(int i) const; - int NumPCBasisParameters() { - return pca.enabled - ? (NumSystematicBasisParameters() - - pca.nrotated_syst_parameters() + pca.npc_parameters()) - : NumSystematicBasisParameters(); + int NumPCBasisParameters() const { + return pca.enabled ? (NumSystematicBasisParameters() - + pca.nrotated_syst_parameters() + pca.npc_parameters()) + : NumSystematicBasisParameters(); } double Chi2(Eigen::ArrayXd const &systematic_vals); - StepProposer MakeProposer(); + StepProposer MakeProposer() const; struct { /// The input root file we read in @@ -106,8 +106,9 @@ struct ParameterList { /// Stores config describing systematics YAML::Node YAMLDoc; } config; -}; -ParameterList MakeFromYAML(YAML::Node const &config); -ParameterList MakeFromYAML(const std::vector &YAMLFiles); -ParameterList MakeFromTMatrix(std::string const &name, std::string const &file); + static ParameterList MakeFromYAML(YAML::Node const &config); + static ParameterList MakeFromYAML(const std::vector &YAMLFiles); + static ParameterList MakeFromTMatrix(std::string const &name, + std::string const &file); +}; diff --git a/Parameters/StepProposer.cpp b/Parameters/StepProposer.cpp index 790327d00..0d3449ccb 100644 --- a/Parameters/StepProposer.cpp +++ b/Parameters/StepProposer.cpp @@ -55,12 +55,11 @@ Eigen::ArrayXd const &StepProposer::Propose() { } void StepProposer::SetProposalMatrix(Eigen::MatrixXd proposal_matrix) { - Eigen::LLT lltproposal(proposal_matrix); + params.proposal = proposal_matrix; + Eigen::LLT lltproposal(params.proposal); if (lltproposal.info() != Eigen::Success) { MACH3LOG_ERROR("Failed to LLT decompose proposal matrix"); throw MaCh3Exception(__FILE__, __LINE__); } params.l_proposal = lltproposal.matrixL(); } - -void StepProposer::Accept() {} diff --git a/Parameters/StepProposer.h b/Parameters/StepProposer.h index 21b89081a..b3c75cccf 100644 --- a/Parameters/StepProposer.h +++ b/Parameters/StepProposer.h @@ -21,6 +21,7 @@ struct StepProposer { Eigen::ArrayXd current, proposed, scale; Eigen::ArrayXi isfree; double global_scale; + Eigen::MatrixXd proposal; Eigen::MatrixXd l_proposal; } params; @@ -40,10 +41,6 @@ struct StepProposer { std::uniform_real_distribution unif; } rng; - struct { - TMatrixDSym proposal; - } root_copies; - struct { bool enabled; size_t nsteps; @@ -53,18 +50,20 @@ struct StepProposer { StepProposer(); StepProposer(Eigen::MatrixXd proposal_matrix, Eigen::ArrayXd current_values); + int NumParameters() const { return int(params.current.size()); } + void SetProposalMatrix(Eigen::MatrixXd proposal_matrix); void SetParameterValues(Eigen::ArrayXd current_values) { params.current = current_values; } - void EnableAdaption(YAML::Node const &adaption_config); - void SetAdaptionCovariance(Eigen::MatrixXd parameter_covariance, - size_t nsteps) { - adaptive.nsteps = nsteps; - adaptive.accepted_parameter_covariance = parameter_covariance; - } + void EnableAdaption(YAML::Node const &){} + // void SetAdaptionCovariance(Eigen::MatrixXd parameter_covariance, + // size_t nsteps) { + // adaptive.nsteps = nsteps; + // adaptive.accepted_parameter_covariance = parameter_covariance; + // } Eigen::ArrayXd const &Propose(); - void Accept(); + void Accept() { params.current = params.proposed; } }; diff --git a/Parameters/test/CMakeLists.txt b/Parameters/test/CMakeLists.txt index 344741244..16ecd1253 100644 --- a/Parameters/test/CMakeLists.txt +++ b/Parameters/test/CMakeLists.txt @@ -13,7 +13,6 @@ add_executable(ParameterListTest ParameterListTest.cpp) target_link_libraries(ParameterListTest PRIVATE Catch2::Catch2WithMain Parameters) catch_discover_tests(ParameterListTest) - -# add_executable(StepProposerTest StepProposerTest.cxx) -# target_link_libraries(StepProposerTest PRIVATE Catch2::Catch2WithMain Parameters) -# catch_discover_tests(StepProposerTest) +add_executable(StepProposerTest StepProposerTest.cpp) +target_link_libraries(StepProposerTest PRIVATE Catch2::Catch2WithMain Parameters) +catch_discover_tests(StepProposerTest) diff --git a/Parameters/test/ParameterListTest.cpp b/Parameters/test/ParameterListTest.cpp index 052db1da3..4b03fd9dc 100644 --- a/Parameters/test/ParameterListTest.cpp +++ b/Parameters/test/ParameterListTest.cpp @@ -8,7 +8,7 @@ using namespace Catch::Matchers; TEST_CASE("MakeFromYAML", "[Construction]") { - auto parlist = MakeFromYAML(YAML::Load( + auto parlist = ParameterList::MakeFromYAML(YAML::Load( R"( Systematics: - Systematic: { @@ -79,7 +79,7 @@ TEST_CASE("MakeFromYAML", "[Construction]") { } TEST_CASE("MakeFromYAML Bad Correlations", "[Construction]") { - REQUIRE_THROWS_AS(MakeFromYAML(YAML::Load( + REQUIRE_THROWS_AS(ParameterList::MakeFromYAML(YAML::Load( R"( Systematics: - Systematic: { @@ -101,7 +101,7 @@ TEST_CASE("MakeFromYAML Bad Correlations", "[Construction]") { )")), MaCh3Exception); - REQUIRE_THROWS_AS(MakeFromYAML(YAML::Load( + REQUIRE_THROWS_AS(ParameterList::MakeFromYAML(YAML::Load( R"( Systematics: - Systematic: { @@ -123,7 +123,7 @@ TEST_CASE("MakeFromYAML Bad Correlations", "[Construction]") { )")), MaCh3Exception); - REQUIRE_THROWS_AS(MakeFromYAML(YAML::Load( + REQUIRE_THROWS_AS(ParameterList::MakeFromYAML(YAML::Load( R"( Systematics: - Systematic: { @@ -162,7 +162,7 @@ TEST_CASE("MakeFromYAML Bad Correlations", "[Construction]") { } TEST_CASE("4param", "[PCA]") { - auto parlist = MakeFromYAML(YAML::Load( + auto parlist = ParameterList::MakeFromYAML(YAML::Load( R"( Systematics: - Systematic: { @@ -175,18 +175,18 @@ TEST_CASE("4param", "[PCA]") { - Systematic: { Names: { FancyName: par2 }, ParameterValues: { PreFitValue: 2 }, - Error: 1, + Error: 2, StepScale: { MCMC: 1 }, ParameterBounds: [ -10, 10 ], - Correlations: [ { par3: 0.9999 }, ] + Correlations: [ { par3: 0.99999 }, ] } - Systematic: { Names: { FancyName: par3 }, ParameterValues: { PreFitValue: 3 }, - Error: 1, + Error: 3, StepScale: { MCMC: 1 }, ParameterBounds: [ -10, 10 ], - Correlations: [ { par2: 0.9999 }, ] + Correlations: [ { par2: 0.99999 }, ] } - Systematic: { Names: { FancyName: par4 }, @@ -206,10 +206,45 @@ TEST_CASE("4param", "[PCA]") { REQUIRE(parlist.pca.nrotated_syst_parameters() == 2); REQUIRE(parlist.pca.npc_parameters() == 1); REQUIRE(parlist.NumPCBasisParameters() == 3); + + REQUIRE(parlist.SystematicParameterIndexToPCIndex(0) == 0); + REQUIRE(parlist.SystematicParameterIndexToPCIndex(1) == + ParameterList::ParameterInPCABlock); + REQUIRE(parlist.SystematicParameterIndexToPCIndex(2) == + ParameterList::ParameterInPCABlock); + REQUIRE(parlist.SystematicParameterIndexToPCIndex(3) == 2); + + Eigen::ArrayXd syst_vals = parlist.params.prefit; + Eigen::ArrayXd pc_vals = Eigen::ArrayXd::Zero(parlist.NumPCBasisParameters()); + + parlist.RotateSystematicParameterValuesToPCBasis(syst_vals, pc_vals); + + // prefit value should get mapped to 0 + REQUIRE(pc_vals[1] == 0); + + parlist.RotatePCParameterValuesToSystematicBasis(pc_vals, syst_vals); + + // prefit value should get mapped and back to prefit + REQUIRE(syst_vals[1] == 2); + REQUIRE(syst_vals[2] == 3); + + // the PC is approximately in the par2+par3 direction, so + // should be able to guess where pc_vals[1] = 1 sets + // syst_vals[1] and syst_vals[2] + pc_vals[1] = 1; + + parlist.RotatePCParameterValuesToSystematicBasis(pc_vals, syst_vals); + + REQUIRE_THAT(syst_vals[1], WithinAbs(4, 1E-4)); + REQUIRE_THAT(syst_vals[2], WithinAbs(6, 1E-4)); + + // check that we can rotate back to the expected value + parlist.RotateSystematicParameterValuesToPCBasis(syst_vals, pc_vals); + REQUIRE_THAT(pc_vals[1], WithinAbs(1, 1E-4)); } TEST_CASE("Out of block correlations", "[PCA]") { - auto parlist = MakeFromYAML(YAML::Load( + auto parlist = ParameterList::MakeFromYAML(YAML::Load( R"( Systematics: - Systematic: { diff --git a/Parameters/test/StepProposerTest.cpp b/Parameters/test/StepProposerTest.cpp index fcab4939f..507ac46e9 100644 --- a/Parameters/test/StepProposerTest.cpp +++ b/Parameters/test/StepProposerTest.cpp @@ -1 +1,119 @@ -StepProposerTest.cpp +#include "Parameters/ParameterList.h" + +#include "catch2/catch_test_macros.hpp" +#include "catch2/matchers/catch_matchers_floating_point.hpp" + +#include + +using namespace Catch::Matchers; + +TEST_CASE("proposal reconstruction", "[Proposal]") { + auto parlist = ParameterList::MakeFromYAML(YAML::Load( + R"( +Systematics: +- Systematic: { + Names: { FancyName: par1 }, + ParameterValues: { PreFitValue: 1 }, + Error: 2.5, + StepScale: { MCMC: 1 }, + ParameterBounds: [ -10, 10 ], + } +- Systematic: { + Names: { FancyName: par2 }, + ParameterValues: { PreFitValue: 2 }, + Error: 2, + StepScale: { MCMC: 1 }, + ParameterBounds: [ -10, 10 ], + Correlations: [ { par3: 0.99999 }, ] + } +- Systematic: { + Names: { FancyName: par3 }, + ParameterValues: { PreFitValue: 3 }, + Error: 3, + StepScale: { MCMC: 1 }, + ParameterBounds: [ -10, 10 ], + Correlations: [ { par2: 0.99999 }, ] + } +- Systematic: { + Names: { FancyName: par4 }, + ParameterValues: { PreFitValue: 4 }, + Error: 9, + StepScale: { MCMC: 1 }, + ParameterBounds: [ -10, 10 ], + } +)")); + + parlist.ConstructTruncatedPCA(1E-4, 1, 2); + + auto proposer = parlist.MakeProposer(); + + REQUIRE(proposer.NumParameters() == parlist.NumPCBasisParameters()); + REQUIRE_FALSE(proposer.special_proposal.enabled); + + Eigen::MatrixXd proposal_reconstructed = + Eigen::MatrixXd::Zero(proposer.NumParameters(), proposer.NumParameters()); + + Eigen::ArrayXd syst_param_vals = + Eigen::ArrayXd::Zero(parlist.NumSystematicBasisParameters()); + Eigen::MatrixXd syst_basis_covariance_reconstructed = + Eigen::MatrixXd::Zero(parlist.NumSystematicBasisParameters(), + parlist.NumSystematicBasisParameters()); + + Eigen::ArrayXd mean_proposed = Eigen::ArrayXd::Zero(proposer.NumParameters()); + Eigen::ArrayXd mean_syst_param_vals = + Eigen::ArrayXd::Zero(parlist.NumSystematicBasisParameters()); + + double nThrows = 1E7; + for (int i = 0; i < nThrows; ++i) { + + // if we never accept a step then they will always we proposed around the + // prefit + Eigen::ArrayXd proposed_vals = proposer.Propose(); + mean_proposed += proposed_vals; + + Eigen::VectorXd pc_basis_delta = + (proposed_vals - proposer.params.current).matrix(); + + proposal_reconstructed += pc_basis_delta * pc_basis_delta.transpose(); + + parlist.RotatePCParameterValuesToSystematicBasis(proposed_vals, + syst_param_vals); + mean_syst_param_vals += syst_param_vals; + Eigen::VectorXd syst_basis_deltas = + (syst_param_vals - parlist.params.prefit).matrix(); + syst_basis_covariance_reconstructed += + syst_basis_deltas * syst_basis_deltas.transpose(); + } + + mean_proposed.array() /= nThrows; + mean_syst_param_vals.array() /= nThrows; + proposal_reconstructed.array() /= nThrows; + syst_basis_covariance_reconstructed.array() /= nThrows; + + REQUIRE_THAT(mean_proposed[0], WithinAbs(1, 1E-2)); + REQUIRE_THAT(mean_proposed[1], WithinAbs(0, 1E-2)); + REQUIRE_THAT(mean_proposed[2], WithinAbs(4, 1E-2)); + + REQUIRE_THAT(mean_syst_param_vals[0], WithinAbs(1, 1E-2)); + REQUIRE_THAT(mean_syst_param_vals[1], WithinAbs(2, 1E-2)); + REQUIRE_THAT(mean_syst_param_vals[2], WithinAbs(3, 1E-2)); + REQUIRE_THAT(mean_syst_param_vals[3], WithinAbs(4, 1E-2)); + + Eigen::MatrixXd proposal_diff = + proposal_reconstructed - proposer.params.proposal; + + for (int i = 0; i < proposal_diff.rows(); ++i) { + for (int j = 0; j < proposal_diff.rows(); ++j) { + REQUIRE_THAT(proposal_diff(i, j), WithinAbs(0, 1E-2)); + } + } + + Eigen::MatrixXd syst_basis_covariance_diff = + syst_basis_covariance_reconstructed - parlist.params.covariance; + + for (int i = 0; i < syst_basis_covariance_diff.rows(); ++i) { + for (int j = 0; j < syst_basis_covariance_diff.rows(); ++j) { + REQUIRE_THAT(syst_basis_covariance_diff(i, j), WithinAbs(0, 1E-2)); + } + } +} From 67c9e365c28109fc45bcb8d30b43b877e0b17b2e Mon Sep 17 00:00:00 2001 From: Luke Pickering Date: Thu, 18 Dec 2025 16:50:42 +0000 Subject: [PATCH 06/14] adds insertparameter to ParameterList --- Parameters/ParameterList.cpp | 126 ++++++++++------- Parameters/ParameterList.h | 28 ++-- Parameters/test/ParameterListTest.cpp | 187 +++++++++++++++++++++++++- 3 files changed, 281 insertions(+), 60 deletions(-) diff --git a/Parameters/ParameterList.cpp b/Parameters/ParameterList.cpp index 4df0421f3..368d313ba 100644 --- a/Parameters/ParameterList.cpp +++ b/Parameters/ParameterList.cpp @@ -3,17 +3,22 @@ #include "Parameters/PCA.h" #include "Parameters/ParameterHandlerUtils.h" -void ParameterList::AddParameters(std::vector const &new_params) { - - for (auto const &p : new_params) { - params.name.push_back(p.name); - params.fancy_name.push_back(p.fancy_name); - params.samples.push_back(p.affected_samples); - params.flip_pivot.push_back(p.flip_pivot); - params.circ_bounds.push_back(p.circ_bounds); +void ParameterList::InsertParameters(int insert_before, + std::vector const &new_params) { + + for (auto it = new_params.rbegin(); it < new_params.rend(); ++it) { + params.name.insert(params.name.begin() + insert_before, it->name); + params.fancy_name.insert(params.fancy_name.begin() + insert_before, + it->fancy_name); + params.samples.insert(params.samples.begin() + insert_before, + it->affected_samples); + params.flip_pivot.insert(params.flip_pivot.begin() + insert_before, + it->flip_pivot); + params.circ_bounds.insert(params.circ_bounds.begin() + insert_before, + it->circ_bounds); } - size_t new_block_start = params.prefit.size(); + auto N = params.prefit.size(); params.prefit.conservativeResize(params.name.size()); params.error.conservativeResize(params.name.size()); @@ -23,21 +28,43 @@ void ParameterList::AddParameters(std::vector const &new_params) { params.flatprior.conservativeResize(params.name.size()); params.isfree.conservativeResize(params.name.size()); - params.covariance.conservativeResize(params.name.size(), params.name.size()); + params.prefit.tail(N - insert_before) = + params.prefit.segment(insert_before, N - insert_before).eval(); + params.error.tail(N - insert_before) = + params.error.segment(insert_before, N - insert_before).eval(); + params.lowbound.tail(N - insert_before) = + params.lowbound.segment(insert_before, N - insert_before).eval(); + params.upbound.tail(N - insert_before) = + params.upbound.segment(insert_before, N - insert_before).eval(); + params.stepscale.tail(N - insert_before) = + params.stepscale.segment(insert_before, N - insert_before).eval(); + params.flatprior.tail(N - insert_before) = + params.flatprior.segment(insert_before, N - insert_before).eval(); + params.isfree.tail(N - insert_before) = + params.isfree.segment(insert_before, N - insert_before).eval(); + + Eigen::MatrixXd ncovariance = + Eigen::MatrixXd::Zero(params.name.size(), params.name.size()); + ncovariance.topLeftCorner(N, N) = params.covariance; + + ncovariance.bottomRightCorner(N - insert_before, N - insert_before) = + ncovariance.block(insert_before, insert_before, N - insert_before, + N - insert_before); + params.covariance = ncovariance; for (size_t i = 0; i < new_params.size(); ++i) { - params.prefit[new_block_start + i] = new_params[i].prefit; - params.error[new_block_start + i] = new_params[i].error; - params.lowbound[new_block_start + i] = new_params[i].bounds[0]; - params.upbound[new_block_start + i] = new_params[i].bounds[1]; - params.stepscale[new_block_start + i] = new_params[i].stepscale; - params.flatprior[new_block_start + i] = new_params[i].flatprior; - params.isfree[new_block_start + i] = new_params[i].isfree; - - params.covariance.row(new_block_start + i).setZero(); - params.covariance.col(new_block_start + i).setZero(); - - params.covariance(new_block_start + i, new_block_start + i) = + params.prefit[insert_before + i] = new_params[i].prefit; + params.error[insert_before + i] = new_params[i].error; + params.lowbound[insert_before + i] = new_params[i].bounds[0]; + params.upbound[insert_before + i] = new_params[i].bounds[1]; + params.stepscale[insert_before + i] = new_params[i].stepscale; + params.flatprior[insert_before + i] = new_params[i].flatprior; + params.isfree[insert_before + i] = new_params[i].isfree; + + params.covariance.row(insert_before + i).setZero(); + params.covariance.col(insert_before + i).setZero(); + + params.covariance(insert_before + i, insert_before + i) = new_params[i].error * new_params[i].error; } @@ -84,12 +111,22 @@ int ParameterList::FindParameter(std::string const &name) const { return int(std::distance(params.name.begin(), pit)); } +int ParameterList::FindParameterByFancyName( + std::string const &fancy_name) const { + auto pit = + std::find(params.fancy_name.begin(), params.fancy_name.end(), fancy_name); + if (pit == params.fancy_name.end()) { + MACH3LOG_ERROR("ParameterList manages no parameter named {}.", fancy_name); + throw MaCh3Exception(__FILE__, __LINE__); + } + return int(std::distance(params.fancy_name.begin(), pit)); +} + void ParameterList::ConstructTruncatedPCA(double threshold, int first, int last) { M3::EnsureNoOutOfBlockCorrelations(params.covariance, {first, last}); - pca.enabled = true; pca.first_index = first; pca.last_index = last; pca.ntail = int(params.prefit.size() - (last + 1)); @@ -150,7 +187,7 @@ void ParameterList::RotateSystematicParameterValuesToPCBasis( } int ParameterList::SystematicParameterIndexToPCIndex(int i) const { - if ((!pca.enabled) || (i < pca.first_index)) { + if (i < pca.first_index) { return i; } else if ((i >= pca.first_index) && (i <= pca.last_index)) { return ParameterInPCABlock; @@ -174,37 +211,32 @@ double ParameterList::Chi2(Eigen::ArrayXd const &systematic_vals) { StepProposer ParameterList::MakeProposer() const { StepProposer proposer; - if (!pca.enabled) { - proposer.SetProposalMatrix(params.covariance); - proposer.SetParameterValues(params.prefit); - } else { - // if we are using a PCA rotation then the proposer should only be aware of - // the PC parameter basis. + // if we are using a PCA rotation then the proposer should only be aware of + // the PC parameter basis. - // copy the unrotated parameters and covariance blocks - int nproposer_parameters = NumPCBasisParameters(); + // copy the unrotated parameters and covariance blocks + int nproposer_parameters = NumPCBasisParameters(); - Eigen::ArrayXd prefit_pc = Eigen::ArrayXd::Zero(nproposer_parameters); + Eigen::ArrayXd prefit_pc = Eigen::ArrayXd::Zero(nproposer_parameters); - RotateSystematicParameterValuesToPCBasis(params.prefit, prefit_pc); + RotateSystematicParameterValuesToPCBasis(params.prefit, prefit_pc); - proposer.SetParameterValues(prefit_pc); + proposer.SetParameterValues(prefit_pc); - Eigen::MatrixXd covariance_pc = - Eigen::MatrixXd::Zero(nproposer_parameters, nproposer_parameters); + Eigen::MatrixXd covariance_pc = + Eigen::MatrixXd::Zero(nproposer_parameters, nproposer_parameters); - covariance_pc.topLeftCorner(pca.first_index, pca.first_index) = - params.covariance.topLeftCorner(pca.first_index, pca.first_index); + covariance_pc.topLeftCorner(pca.first_index, pca.first_index) = + params.covariance.topLeftCorner(pca.first_index, pca.first_index); - covariance_pc.block(pca.first_index, pca.first_index, pca.npc_parameters(), - pca.npc_parameters()) = - Eigen::MatrixXd::Identity(pca.npc_parameters(), pca.npc_parameters()); + covariance_pc.block(pca.first_index, pca.first_index, pca.npc_parameters(), + pca.npc_parameters()) = + Eigen::MatrixXd::Identity(pca.npc_parameters(), pca.npc_parameters()); - covariance_pc.bottomRightCorner(pca.ntail, pca.ntail) = - params.covariance.bottomRightCorner(pca.ntail, pca.ntail); + covariance_pc.bottomRightCorner(pca.ntail, pca.ntail) = + params.covariance.bottomRightCorner(pca.ntail, pca.ntail); - proposer.SetProposalMatrix(covariance_pc); - } + proposer.SetProposalMatrix(covariance_pc); // PCA parameters get a step scale of 1 proposer.params.scale = Eigen::ArrayXd::Ones(NumPCBasisParameters()); @@ -444,7 +476,6 @@ ParameterList ParameterList::MakeFromYAML(YAML::Node const &config) { parlist.params.covariance = M3::MakeMatrixPosDef(parlist.params.covariance); - parlist.pca.enabled = false; parlist.pca.first_index = parlist.NumSystematicBasisParameters(); parlist.pca.last_index = parlist.NumSystematicBasisParameters(); parlist.pca.ntail = 0; @@ -527,7 +558,6 @@ ParameterList ParameterList::MakeFromTMatrix(std::string const &name, parlist.params.covariance = M3::MakeMatrixPosDef(M3::ROOTToEigen(*CovMat)); - parlist.pca.enabled = false; parlist.pca.first_index = parlist.NumSystematicBasisParameters(); parlist.pca.last_index = parlist.NumSystematicBasisParameters(); parlist.pca.ntail = 0; diff --git a/Parameters/ParameterList.h b/Parameters/ParameterList.h index e9a427678..852c9d452 100644 --- a/Parameters/ParameterList.h +++ b/Parameters/ParameterList.h @@ -51,11 +51,19 @@ struct ParameterList { std::tuple circ_bounds; }; - void AddParameters(std::vector const &new_params); + void InsertParameters(int insert_at, + std::vector const &new_params); + void InsertParameter(int insert_at, ParamInfo const &new_param) { + InsertParameters(insert_at, { + new_param, + }); + } + + void AddParameters(std::vector const &new_params) { + InsertParameters(NumSystematicBasisParameters(), new_params); + } void AddParameter(ParamInfo const &new_param) { - AddParameters({ - new_param, - }); + InsertParameter(NumSystematicBasisParameters(), new_param); } std::string SystematicParameterToString(int i) const; @@ -65,10 +73,14 @@ struct ParameterList { int paramid, std::map const &correlations); int FindParameter(std::string const &name) const; + int FindParameterByFancyName(std::string const &fancy_name) const; int NumSystematicBasisParameters() const { return int(params.prefit.size()); } + int NumPCBasisParameters() const { + return NumSystematicBasisParameters() - pca.nrotated_syst_parameters() + + pca.npc_parameters(); + } struct { - bool enabled; int first_index, last_index, ntail; Eigen::MatrixXd pc_to_syst_rotation; Eigen::MatrixXd syst_to_pc_rotation; @@ -90,12 +102,6 @@ struct ParameterList { constexpr static int ParameterInPCABlock = std::numeric_limits::max(); int SystematicParameterIndexToPCIndex(int i) const; - int NumPCBasisParameters() const { - return pca.enabled ? (NumSystematicBasisParameters() - - pca.nrotated_syst_parameters() + pca.npc_parameters()) - : NumSystematicBasisParameters(); - } - double Chi2(Eigen::ArrayXd const &systematic_vals); StepProposer MakeProposer() const; diff --git a/Parameters/test/ParameterListTest.cpp b/Parameters/test/ParameterListTest.cpp index 4b03fd9dc..f62901517 100644 --- a/Parameters/test/ParameterListTest.cpp +++ b/Parameters/test/ParameterListTest.cpp @@ -161,6 +161,112 @@ TEST_CASE("MakeFromYAML Bad Correlations", "[Construction]") { MaCh3Exception); } +TEST_CASE("InsertParameters", "[Construction]") { + auto parlist = ParameterList::MakeFromYAML(YAML::Load( + R"( +Systematics: +- Systematic: { + Names: { FancyName: par1 }, + ParameterValues: { PreFitValue: 1 }, + Error: 1, + StepScale: { MCMC: 0.1 }, + ParameterBounds: [ -3, 3 ], + } +- Systematic: { + Names: { FancyName: par2 }, + ParameterValues: { PreFitValue: 2 }, + Error: 1, + StepScale: { MCMC: 0.1 }, + ParameterBounds: [ -3, 3 ], + } +- Systematic: { + Names: { FancyName: par3 }, + ParameterValues: { PreFitValue: 3 }, + Error: 0.5, + StepScale: { MCMC: 1 }, + ParameterBounds: [ -3, 3 ], + } +)")); + + REQUIRE(parlist.FindParameter("par1") == 0); + REQUIRE(parlist.FindParameter("par2") == 1); + REQUIRE(parlist.FindParameter("par3") == 2); + + parlist.AddParameter({"extraPar", + "extraPar", + 5, + 5, + 1, + {-5, 5}, + false, + true, + {}, + {false, 0}, + {false, 0, 0}}); + + REQUIRE(parlist.FindParameter("par1") == 0); + REQUIRE(parlist.FindParameter("par2") == 1); + REQUIRE(parlist.FindParameter("par3") == 2); + REQUIRE(parlist.FindParameter("extraPar") == 3); + + parlist.InsertParameter(0, {"extraPar2", + "extraPar2", + 4, + 4, + 1, + {-4, 4}, + false, + true, + {}, + {false, 0}, + {false, 0, 0}}); + + REQUIRE(parlist.FindParameter("extraPar2") == 0); + REQUIRE(parlist.FindParameter("par1") == 1); + REQUIRE(parlist.FindParameter("par2") == 2); + REQUIRE(parlist.FindParameter("par3") == 3); + REQUIRE(parlist.FindParameter("extraPar") == 4); + + parlist.InsertParameter(2, {"extraPar3", + "extraPar3", + 7, + 7, + 1, + {-7, 7}, + false, + true, + {}, + {false, 0}, + {false, 0, 0}}); + + REQUIRE(parlist.FindParameter("extraPar2") == 0); + REQUIRE(parlist.FindParameter("par1") == 1); + REQUIRE(parlist.FindParameter("extraPar3") == 2); + REQUIRE(parlist.FindParameter("par2") == 3); + REQUIRE(parlist.FindParameter("par3") == 4); + REQUIRE(parlist.FindParameter("extraPar") == 5); + + parlist.InsertParameter(6, {"extraPar4", + "extraPar4", + 8, + 8, + 1, + {-8, 8}, + false, + true, + {}, + {false, 0}, + {false, 0, 0}}); + + REQUIRE(parlist.FindParameter("extraPar2") == 0); + REQUIRE(parlist.FindParameter("par1") == 1); + REQUIRE(parlist.FindParameter("extraPar3") == 2); + REQUIRE(parlist.FindParameter("par2") == 3); + REQUIRE(parlist.FindParameter("par3") == 4); + REQUIRE(parlist.FindParameter("extraPar") == 5); + REQUIRE(parlist.FindParameter("extraPar4") == 6); +} + TEST_CASE("4param", "[PCA]") { auto parlist = ParameterList::MakeFromYAML(YAML::Load( R"( @@ -199,7 +305,7 @@ TEST_CASE("4param", "[PCA]") { parlist.ConstructTruncatedPCA(1E-4, 1, 2); - REQUIRE(parlist.pca.enabled); + REQUIRE(parlist.NumSystematicBasisParameters() == 4); REQUIRE(parlist.pca.first_index == 1); REQUIRE(parlist.pca.last_index == 2); REQUIRE(parlist.pca.ntail == 1); @@ -243,6 +349,85 @@ TEST_CASE("4param", "[PCA]") { REQUIRE_THAT(pc_vals[1], WithinAbs(1, 1E-4)); } +TEST_CASE("no explicit PCA checks", "[PCA]") { + auto parlist = ParameterList::MakeFromYAML(YAML::Load( + R"( +Systematics: +- Systematic: { + Names: { FancyName: par1 }, + ParameterValues: { PreFitValue: 1 }, + Error: 1, + StepScale: { MCMC: 1 }, + ParameterBounds: [ -10, 10 ], + } +- Systematic: { + Names: { FancyName: par2 }, + ParameterValues: { PreFitValue: 2 }, + Error: 2, + StepScale: { MCMC: 1 }, + ParameterBounds: [ -10, 10 ], + Correlations: [ { par3: 0.99999 }, ] + } +- Systematic: { + Names: { FancyName: par3 }, + ParameterValues: { PreFitValue: 3 }, + Error: 3, + StepScale: { MCMC: 1 }, + ParameterBounds: [ -10, 10 ], + Correlations: [ { par2: 0.99999 }, ] + } +- Systematic: { + Names: { FancyName: par4 }, + ParameterValues: { PreFitValue: 4 }, + Error: 1, + StepScale: { MCMC: 1 }, + ParameterBounds: [ -10, 10 ], + } +)")); + + REQUIRE(parlist.NumSystematicBasisParameters() == 4); + REQUIRE(parlist.pca.first_index == 4); + REQUIRE(parlist.pca.last_index == 4); + REQUIRE(parlist.pca.ntail == 0); + REQUIRE(parlist.pca.nrotated_syst_parameters() == 0); + REQUIRE(parlist.pca.npc_parameters() == 0); + REQUIRE(parlist.NumPCBasisParameters() == 4); + + Eigen::ArrayXd syst_vals = parlist.params.prefit; + Eigen::ArrayXd pc_vals = Eigen::ArrayXd::Zero(parlist.NumPCBasisParameters()); + + parlist.RotateSystematicParameterValuesToPCBasis(syst_vals, pc_vals); + + REQUIRE(pc_vals[0] == syst_vals[0]); + REQUIRE(pc_vals[1] == syst_vals[1]); + REQUIRE(pc_vals[2] == syst_vals[2]); + REQUIRE(pc_vals[3] == syst_vals[3]); + + parlist.RotatePCParameterValuesToSystematicBasis(pc_vals, syst_vals); + + // prefit value should get mapped and back to prefit + REQUIRE(syst_vals[0] == 1); + REQUIRE(syst_vals[1] == 2); + REQUIRE(syst_vals[2] == 3); + REQUIRE(syst_vals[3] == 4); + + // the PC is approximately in the par2+par3 direction, so + // should be able to guess where pc_vals[1] = 1 sets + // syst_vals[1] and syst_vals[2] + pc_vals[1] = 1; + + parlist.RotatePCParameterValuesToSystematicBasis(pc_vals, syst_vals); + + REQUIRE(syst_vals[0] == 1); + REQUIRE(syst_vals[1] == 1); + REQUIRE(syst_vals[2] == 3); + REQUIRE(syst_vals[3] == 4); + + // check that we can rotate back to the expected value + parlist.RotateSystematicParameterValuesToPCBasis(syst_vals, pc_vals); + REQUIRE(pc_vals[1] == 1); +} + TEST_CASE("Out of block correlations", "[PCA]") { auto parlist = ParameterList::MakeFromYAML(YAML::Load( R"( From 64fc8bed4dfcb00409d489d150c0c9656082e04c Mon Sep 17 00:00:00 2001 From: Luke Pickering Date: Thu, 18 Dec 2025 16:51:02 +0000 Subject: [PATCH 07/14] tidy up ParameterHandlerGeneric --- Parameters/ParameterHandlerGeneric.cpp | 1044 +++++++++++++----------- Parameters/ParameterHandlerGeneric.h | 528 ++++++------ Parameters/ParameterStructs.h | 395 +++++---- Samples/SampleHandlerFD.cpp | 43 +- 4 files changed, 1105 insertions(+), 905 deletions(-) diff --git a/Parameters/ParameterHandlerGeneric.cpp b/Parameters/ParameterHandlerGeneric.cpp index 770b7b330..225fe1fcd 100644 --- a/Parameters/ParameterHandlerGeneric.cpp +++ b/Parameters/ParameterHandlerGeneric.cpp @@ -4,16 +4,18 @@ // ETA - YAML constructor // this will replace the root file constructor but let's keep it in // to do some validations -ParameterHandlerGeneric::ParameterHandlerGeneric(const std::vector& YAMLFile, std::string name, double threshold, int FirstPCA, int LastPCA) - : ParameterHandlerBase(YAMLFile, name, threshold, FirstPCA, LastPCA){ -// ******************************************** +ParameterHandlerGeneric::ParameterHandlerGeneric( + const std::vector &YAMLFile, std::string name, + double threshold, int FirstPCA, int LastPCA) + : ParameterHandlerBase(YAMLFile, name, threshold, FirstPCA, LastPCA) { + // ******************************************** InitParametersTypeFromConfig(); - //ETA - again this really doesn't need to be hear... - for (int i = 0; i < _fNumPar; i++) - { + // ETA - again this really doesn't need to be here... + for (int i = 0; i < parlist.NumSystematicBasisParameters(); i++) { // Sort out the print length - if(int(_fNames[i].length()) > PrintLength) PrintLength = int(_fNames[i].length()); + if (int(parlist.params.name[i].length()) > PrintLength) + PrintLength = int(parlist.params.name[i].length()); } // end the for loop MACH3LOG_DEBUG("Constructing instance of ParameterHandler"); @@ -24,117 +26,143 @@ ParameterHandlerGeneric::ParameterHandlerGeneric(const std::vector& // ******************************************** void ParameterHandlerGeneric::InitParametersTypeFromConfig() { -// ******************************************** - _fSystToGlobalSystIndexMap.resize(SystType::kSystTypes); - - _fParamType = std::vector(_fNumPar); - _ParameterGroup = std::vector(_fNumPar); - - //KS: We know at most how params we expect so reserve memory for max possible params. Later we will shrink to size to not waste memory. Reserving means slightly faster loading and possible less memory fragmentation. - NormParams.reserve(_fNumPar); - SplineParams.reserve(_fNumPar); - FuncParams.reserve(_fNumPar); - OscParams.reserve(_fNumPar); - - int i = 0; - unsigned int ParamCounter[SystType::kSystTypes] = {0}; - //ETA - read in the systematics. Would be good to add in some checks to make sure - //that there are the correct number of entries i.e. are the _fNumPars for Names, - //PreFitValues etc etc. - for (auto const ¶m : _fYAMLDoc["Systematics"]) - { - _ParameterGroup[i] = Get(param["Systematic"]["ParameterGroup"], __FILE__ , __LINE__); - - //Fill the map to get the correlations later as well - auto ParamType = Get(param["Systematic"]["Type"], __FILE__ , __LINE__); - //Now load in variables for spline systematics only - if (ParamType.find(SystType_ToString(SystType::kSpline)) != std::string::npos) - { - //Set param type - _fParamType[i] = SystType::kSpline; - // Fill Spline info - SplineParams.push_back(GetSplineParameter(param["Systematic"], i)); - - if (param["Systematic"]["SplineInformation"]["SplineName"]) { - _fSplineNames.push_back(param["Systematic"]["SplineInformation"]["SplineName"].as()); - } + // ******************************************** + + // ETA - read in the systematics. Would be good to add in some checks to make + // sure that there are the correct number of entries i.e. are the _fNumPars + // for Names, PreFitValues etc etc. + for (auto const &node : _fYAMLDoc["Systematics"]) { + auto const &pardef = node["Systematic"]; + + auto group = Get(pardef["ParameterGroup"], __FILE__, __LINE__); + auto fancy_name = + Get(pardef["Names"]["FancyName"], __FILE__, __LINE__); + + // Fill the map to get the correlations later as well + auto ParamType = + String_ToSystType(Get(pardef["Type"], __FILE__, __LINE__)); + + // Now load in variables for spline systematics only + switch (ParamType) { + case kSpline: { + SplineParams.push_back(GetSplineParameter(pardef["SplineInformation"])); + SplineParams.back().group = group; + SplineParams.back().fancy_name = fancy_name; + break; + } + case kNorm: { + NormParams.push_back(GetNormParameter(pardef)); + NormParams.back().group = group; + NormParams.back().fancy_name = fancy_name; + break; + } + case kFunc: { + FuncParams.push_back(GetFunctionalParameters(pardef)); + FuncParams.back().group = group; + FuncParams.back().fancy_name = fancy_name; + break; + } + case kOsc: { + OscParams.push_back(GetOscillationParameters(pardef)); + OscParams.back().group = group; + OscParams.back().fancy_name = fancy_name; + break; + } + case kSystTypes: + default: { + break; + } + } - //Insert the mapping from the spline index i.e. the length of _fSplineNames etc - //to the Systematic index i.e. the counter for things like _fSampleID - _fSystToGlobalSystIndexMap[SystType::kSpline].insert(std::make_pair(ParamCounter[SystType::kSpline], i)); - ParamCounter[SystType::kSpline]++; - } else if(param["Systematic"]["Type"].as() == SystType_ToString(SystType::kNorm)) { - _fParamType[i] = SystType::kNorm; - NormParams.push_back(GetNormParameter(param["Systematic"], i)); - _fSystToGlobalSystIndexMap[SystType::kNorm].insert(std::make_pair(ParamCounter[SystType::kNorm], i)); - ParamCounter[SystType::kNorm]++; - } else if(param["Systematic"]["Type"].as() == SystType_ToString(SystType::kFunc)){ - _fParamType[i] = SystType::kFunc; - FuncParams.push_back(GetFunctionalParameters(param["Systematic"], i)); - _fSystToGlobalSystIndexMap[SystType::kFunc].insert(std::make_pair(ParamCounter[SystType::kFunc], i)); - ParamCounter[SystType::kFunc]++; - } else if(param["Systematic"]["Type"].as() == SystType_ToString(SystType::kOsc)){ - _fParamType[i] = SystType::kOsc; - OscParams.push_back(GetOscillationParameters(param["Systematic"], i)); - _fSystToGlobalSystIndexMap[SystType::kOsc].insert(std::make_pair(ParamCounter[SystType::kOsc], i)); - ParamCounter[SystType::kOsc]++; - } else{ - MACH3LOG_ERROR("Given unrecognised systematic type: {}", param["Systematic"]["Type"].as()); - std::string expectedTypes = "Expecting "; - for (int s = 0; s < SystType::kSystTypes; ++s) { - if (s > 0) expectedTypes += ", "; - expectedTypes += SystType_ToString(static_cast(s)) + "\""; - } - expectedTypes += "."; - MACH3LOG_ERROR(expectedTypes); + } // end loop over params + + DetermineGlobalParameterIndices(); +} + +void ParameterHandlerGeneric::DetermineGlobalParameterIndices() { + + GlobalParams.clear(); + for (auto &par : SplineParams) { + par.index = parlist.FindParameterByFancyName(par.fancy_name); + GlobalParams.push_back(&par); + } + for (auto &par : NormParams) { + par.index = parlist.FindParameterByFancyName(par.fancy_name); + GlobalParams.push_back(&par); + + if (GetLowerBound(par.index) < 0.) { + MACH3LOG_ERROR("Normalisation Parameter {} ({}), has lower parameters " + "bound which can go below 0 and is equal {}", + GetParFancyName(par.index), par.index, + GetLowerBound(par.index)); + MACH3LOG_ERROR( + "Normalisation parameters can't go bellow 0 as this is unphysical"); throw MaCh3Exception(__FILE__, __LINE__); } - i++; - } //end loop over params + } + for (auto &par : FuncParams) { + par.index = parlist.FindParameterByFancyName(par.fancy_name); + par.valuePtr = RetPointer(par.index); + GlobalParams.push_back(&par); + } + for (auto &par : OscParams) { + par.index = parlist.FindParameterByFancyName(par.fancy_name); + par.valuePtr = RetPointer(par.index); + GlobalParams.push_back(&par); + } - //Add a sanity check, - if(_fSplineNames.size() != ParamCounter[SystType::kSpline]){ - MACH3LOG_ERROR("_fSplineNames is of size {} but found {} spline parameters", _fSplineNames.size(), ParamCounter[SystType::kSpline]); - throw MaCh3Exception(__FILE__, __LINE__); + //LP order by index... probably doesn't help but probably doesn't hurt + std::sort(GlobalParams.begin(), GlobalParams.end(), + [](TypeParameterBase const *l, TypeParameterBase const *r) { + return l->index < r->index; + }); +} + +TypeParameterBase const *ParameterHandlerGeneric::GetParam(const int i) const { + auto it = std::find_if( + GlobalParams.begin(), GlobalParams.end(), + [=](TypeParameterBase const *par) { return par->index == i; }); + if (it == GlobalParams.end()) { + return nullptr; } - //KS We resized them above to all params to fight memory fragmentation, now let's resize to fit only allocated memory to save RAM - NormParams.shrink_to_fit(); - SplineParams.shrink_to_fit(); - FuncParams.shrink_to_fit(); - OscParams.shrink_to_fit(); + return *it; +} + +SystType ParameterHandlerGeneric::GetParamType(const int i) const { + auto par = GetParam(i); + return par ? par->syst_type : kSystTypes; } // ******************************************** ParameterHandlerGeneric::~ParameterHandlerGeneric() { -// ******************************************** - MACH3LOG_DEBUG("Deleting ParameterHandler"); + // ******************************************** + MACH3LOG_DEBUG("Destructing ParameterHandler"); } // ******************************************** // DB Grab the Spline Names for the relevant SampleName -const std::vector ParameterHandlerGeneric::GetSplineParsNamesFromSampleName(const std::string& SampleName) { -// ******************************************** +const std::vector +ParameterHandlerGeneric::GetSplineParsNamesFromSampleName( + const std::string &SampleName) { + // ******************************************** std::vector returnVec; - for (auto &pair : _fSystToGlobalSystIndexMap[SystType::kSpline]) { - auto &SplineIndex = pair.first; - auto &SystIndex = pair.second; - if (AppliesToSample(SystIndex, SampleName)) { //If parameter applies to required Sample - returnVec.push_back(_fSplineNames.at(SplineIndex)); + for (auto &sp : SplineParams) { + if (AppliesToSample(sp.index, SampleName)) { + returnVec.push_back(sp.spline_name); } } return returnVec; } // ******************************************** -const std::vector ParameterHandlerGeneric::GetSplineInterpolationFromSampleName(const std::string& SampleName) { -// ******************************************** +const std::vector +ParameterHandlerGeneric::GetSplineInterpolationFromSampleName( + const std::string &SampleName) { + // ******************************************** std::vector returnVec; - for (auto &pair : _fSystToGlobalSystIndexMap[SystType::kSpline]) { - auto &SplineIndex = pair.first; - auto &SystIndex = pair.second; - - if (AppliesToSample(SystIndex, SampleName)) { //If parameter applies to required SampleID - returnVec.push_back(SplineParams.at(SplineIndex)._SplineInterpolationType); + for (auto &sp : SplineParams) { + if (AppliesToSample(sp.index, SampleName)) { + returnVec.push_back(sp.SplineInterpolationType); } } return returnVec; @@ -142,16 +170,14 @@ const std::vector ParameterHandlerGeneric::GetSplineInterpo // ******************************************** // DB Grab the Spline Modes for the relevant SampleName -const std::vector< std::vector > ParameterHandlerGeneric::GetSplineModeVecFromSampleName(const std::string& SampleName) { -// ******************************************** - std::vector< std::vector > returnVec; - //Need a counter or something to correctly get the index in _fSplineModes since it's not of length nPars - //Should probably just make a std::map for param name to FD spline index - for (auto &pair : _fSystToGlobalSystIndexMap[SystType::kSpline]) { - auto &SplineIndex = pair.first; - auto &SystIndex = pair.second; - if (AppliesToSample(SystIndex, SampleName)) { //If parameter applies to required SampleID - returnVec.push_back(SplineParams.at(SplineIndex)._fSplineModes); +const std::vector> +ParameterHandlerGeneric::GetSplineModeVecFromSampleName( + const std::string &SampleName) { + // ******************************************** + std::vector> returnVec; + for (auto &sp : SplineParams) { + if (AppliesToSample(sp.index, SampleName)) { + returnVec.push_back(sp.SplineModes); } } return returnVec; @@ -159,100 +185,76 @@ const std::vector< std::vector > ParameterHandlerGeneric::GetSplineModeVecF // ******************************************** // Get Norm params -NormParameter ParameterHandlerGeneric::GetNormParameter(const YAML::Node& param, const int Index) { -// ******************************************** +NormParameter +ParameterHandlerGeneric::GetNormParameter(const YAML::Node ¶m) { + // ******************************************** NormParameter norm; - GetBaseParameter(param, Index, norm); - /// ETA size 0 to mean apply to all /// Ultimately all this information ends up in the @NormParams vector - norm.modes = GetFromManager>(param["Mode"], {}, __FILE__ , __LINE__); - norm.pdgs = GetFromManager>(param["NeutrinoFlavour"], {}, __FILE__ , __LINE__); - norm.preoscpdgs = GetFromManager>(param["NeutrinoFlavourUnosc"], {}, __FILE__ , __LINE__); - norm.targets = GetFromManager>(param["TargetNuclei"], {}, __FILE__ , __LINE__); - - if(_fLowBound[Index] < 0.) { - MACH3LOG_ERROR("Normalisation Parameter {} ({}), has lower parameters bound which can go below 0 and is equal {}", - GetParFancyName(Index), Index, _fLowBound[Index]); - MACH3LOG_ERROR("Normalisation parameters can't go bellow 0 as this is unphysical"); - throw MaCh3Exception(__FILE__, __LINE__); - } + norm.modes = + GetFromManager>(param["Mode"], {}, __FILE__, __LINE__); + norm.pdgs = GetFromManager>(param["NeutrinoFlavour"], {}, + __FILE__, __LINE__); + norm.preoscpdgs = GetFromManager>( + param["NeutrinoFlavourUnosc"], {}, __FILE__, __LINE__); + norm.targets = GetFromManager>(param["TargetNuclei"], {}, + __FILE__, __LINE__); int NumKinematicCuts = 0; - if(param["KinematicCuts"]) { + if (param["KinematicCuts"]) { NumKinematicCuts = int(param["KinematicCuts"].size()); std::vector TempKinematicStrings; std::vector>> TempKinematicBounds; - //First element of TempKinematicBounds is always -999, and size is then 3 - for(int KinVar_i = 0 ; KinVar_i < NumKinematicCuts ; ++KinVar_i) { - //ETA: This is a bit messy, Kinematic cuts is a list of maps - for (YAML::const_iterator it = param["KinematicCuts"][KinVar_i].begin();it!=param["KinematicCuts"][KinVar_i].end();++it) { + // First element of TempKinematicBounds is always -999, and size is then 3 + for (int KinVar_i = 0; KinVar_i < NumKinematicCuts; ++KinVar_i) { + // ETA: This is a bit messy, Kinematic cuts is a list of maps + for (YAML::const_iterator it = param["KinematicCuts"][KinVar_i].begin(); + it != param["KinematicCuts"][KinVar_i].end(); ++it) { TempKinematicStrings.push_back(it->first.as()); TempKinematicBounds.push_back(Get2DBounds(it->second)); } - if(TempKinematicStrings.size() == 0) { - MACH3LOG_ERROR("Received a KinematicCuts node but couldn't read the contents (it's a list of single-element dictionaries (python) = map of pairs (C++))"); - MACH3LOG_ERROR("For Param {}", norm.name); + if (TempKinematicStrings.size() == 0) { + MACH3LOG_ERROR("Received a KinematicCuts node but couldn't read the " + "contents (it's a list of single-element dictionaries " + "(python) = map of pairs (C++))"); + MACH3LOG_ERROR("For Param {}", norm.fancy_name); throw MaCh3Exception(__FILE__, __LINE__); } - }//KinVar_i + } // KinVar_i norm.KinematicVarStr = TempKinematicStrings; norm.Selection = TempKinematicBounds; } - //Next ones are kinematic bounds on where normalisation parameter should apply - //We set a bool to see if any bounds exist so we can short-circuit checking all of them every step + // Next ones are kinematic bounds on where normalisation parameter should + // apply We set a bool to see if any bounds exist so we can short-circuit + // checking all of them every step bool HasKinBounds = false; - if(norm.KinematicVarStr.size() > 0) HasKinBounds = true; + if (norm.KinematicVarStr.size() > 0) + HasKinBounds = true; norm.hasKinBounds = HasKinBounds; - //End of kinematic bound checking + // End of kinematic bound checking return norm; } -// ******************************************** -// Get Base Param -void ParameterHandlerGeneric::GetBaseParameter(const YAML::Node& param, const int Index, TypeParameterBase& Parameter) { -// ******************************************** - // KS: For now we don't use so avoid compilation error - (void) param; - - Parameter.name = GetParFancyName(Index); - - // Set the global parameter index of the normalisation parameter - Parameter.index = Index; -} - - // ******************************************** // Grab the global syst index for the relevant SampleName -// i.e. get a vector of size nSplines where each entry is filled with the global syst number -const std::vector ParameterHandlerGeneric::GetGlobalSystIndexFromSampleName(const std::string& SampleName, const SystType Type) { -// ******************************************** +// i.e. get a vector of size nSplines where each entry is filled with the global +// syst number +const std::vector +ParameterHandlerGeneric::GetGlobalSystIndexFromSampleName( + const std::string &SampleName, const SystType Type) { + // ******************************************** std::vector returnVec; - for (auto &pair : _fSystToGlobalSystIndexMap[Type]) { - auto &SystIndex = pair.second; - if (AppliesToSample(SystIndex, SampleName)) { //If parameter applies to required SampleID - returnVec.push_back(SystIndex); + for (auto par : GlobalParams) { + if (par->syst_type != Type) { + continue; } - } - return returnVec; -} - -// ******************************************** -// Grab the global syst index for the relevant SampleName -// i.e. get a vector of size nSplines where each entry is filled with the global syst number -const std::vector ParameterHandlerGeneric::GetSystIndexFromSampleName(const std::string& SampleName, const SystType Type) const { -// ******************************************** - std::vector returnVec; - for (auto &pair : _fSystToGlobalSystIndexMap[Type]) { - auto &SplineIndex = pair.first; - auto &systIndex = pair.second; - if (AppliesToSample(systIndex, SampleName)) { //If parameter applies to required SampleID - returnVec.push_back(SplineIndex); + if (AppliesToSample(par->index, SampleName)) { + returnVec.push_back(par->index); } } return returnVec; @@ -260,114 +262,142 @@ const std::vector ParameterHandlerGeneric::GetSystIndexFromSampleName(const // ******************************************** // Get Norm params -SplineParameter ParameterHandlerGeneric::GetSplineParameter(const YAML::Node& param, const int Index) { -// ******************************************** +SplineParameter +ParameterHandlerGeneric::GetSplineParameter(const YAML::Node ¶m) { + // ******************************************** + SplineParameter Spline; - GetBaseParameter(param, Index, Spline); - //Now get the Spline interpolation type - if (param["SplineInformation"]["InterpolationType"]){ - for(int InterpType = 0; InterpType < kSplineInterpolations ; ++InterpType){ - if(param["SplineInformation"]["InterpolationType"].as() == SplineInterpolation_ToString(SplineInterpolation(InterpType))) - Spline._SplineInterpolationType = SplineInterpolation(InterpType); + // Now get the Spline interpolation type + if (param["InterpolationType"]) { + for (int InterpType = 0; InterpType < kSplineInterpolations; ++InterpType) { + if (param["InterpolationType"].as() == + SplineInterpolation_ToString(SplineInterpolation(InterpType))) + Spline.SplineInterpolationType = SplineInterpolation(InterpType); } - } else { //KS: By default use TSpline3 - Spline._SplineInterpolationType = kTSpline3; + } else { // KS: By default use TSpline3 + Spline.SplineInterpolationType = kTSpline3; + } + + Spline.SplineKnotUpBound = GetFromManager( + param["SplineKnotUpBound"], M3::DefSplineKnotUpBound, __FILE__, __LINE__); + Spline.SplineKnotLowBound = + GetFromManager(param["SplineKnotLowBound"], + M3::DefSplineKnotLowBound, __FILE__, __LINE__); + + if (Spline.SplineKnotUpBound != M3::DefSplineKnotUpBound || + Spline.SplineKnotLowBound != M3::DefSplineKnotLowBound) { + MACH3LOG_WARN( + "Spline knot capping enabled with bounds [{}, {}]. For reliable fits, " + "consider modifying the input generation instead.", + Spline.SplineKnotLowBound, Spline.SplineKnotUpBound); } - Spline._SplineKnotUpBound = GetFromManager(param["SplineInformation"]["SplineKnotUpBound"], M3::DefSplineKnotUpBound, __FILE__ , __LINE__); - Spline._SplineKnotLowBound = GetFromManager(param["SplineInformation"]["SplineKnotLowBound"], M3::DefSplineKnotLowBound, __FILE__ , __LINE__); + // If there is no mode information given then this will be an empty vector + Spline.SplineModes = + GetFromManager(param["Mode"], std::vector(), __FILE__, __LINE__); - if(Spline._SplineKnotUpBound != M3::DefSplineKnotUpBound || Spline._SplineKnotLowBound != M3::DefSplineKnotLowBound) { - MACH3LOG_WARN("Spline knot capping enabled with bounds [{}, {}]. For reliable fits, consider modifying the input generation instead.", - Spline._SplineKnotLowBound, Spline._SplineKnotUpBound); + if (param["SplineName"]) { + Spline.spline_name = param["SplineName"].as(); } - //If there is no mode information given then this will be an empty vector - Spline._fSplineModes = GetFromManager(param["SplineInformation"]["Mode"], std::vector(), __FILE__ , __LINE__); return Spline; } // ******************************************** // Get Func params -FunctionalParameter ParameterHandlerGeneric::GetFunctionalParameters(const YAML::Node& param, const int Index) { -// ******************************************** +FunctionalParameter +ParameterHandlerGeneric::GetFunctionalParameters(const YAML::Node ¶m) { + // ******************************************** FunctionalParameter func; - GetBaseParameter(param, Index, func); - func.pdgs = GetFromManager>(param["NeutrinoFlavour"], std::vector(), __FILE__ , __LINE__); - func.targets = GetFromManager>(param["TargetNuclei"], std::vector(), __FILE__ , __LINE__); - func.modes = GetFromManager>(param["Mode"], std::vector(), __FILE__ , __LINE__); - func.preoscpdgs = GetFromManager>(param["NeutrinoFlavourUnosc"], std::vector(), __FILE__ , __LINE__); + func.pdgs = GetFromManager>( + param["NeutrinoFlavour"], std::vector(), __FILE__, __LINE__); + func.targets = GetFromManager>( + param["TargetNuclei"], std::vector(), __FILE__, __LINE__); + func.modes = GetFromManager>( + param["Mode"], std::vector(), __FILE__, __LINE__); + func.preoscpdgs = GetFromManager>( + param["NeutrinoFlavourUnosc"], std::vector(), __FILE__, __LINE__); // HH - Copied from GetXsecNorm int NumKinematicCuts = 0; - if(param["KinematicCuts"]){ + if (param["KinematicCuts"]) { NumKinematicCuts = int(param["KinematicCuts"].size()); std::vector TempKinematicStrings; std::vector>> TempKinematicBounds; - //First element of TempKinematicBounds is always -999, and size is then 3 - for(int KinVar_i = 0 ; KinVar_i < NumKinematicCuts ; ++KinVar_i){ - //ETA: This is a bit messy, Kinematic cuts is a list of maps - for (YAML::const_iterator it = param["KinematicCuts"][KinVar_i].begin();it!=param["KinematicCuts"][KinVar_i].end();++it) { + // First element of TempKinematicBounds is always -999, and size is then 3 + for (int KinVar_i = 0; KinVar_i < NumKinematicCuts; ++KinVar_i) { + // ETA: This is a bit messy, Kinematic cuts is a list of maps + for (YAML::const_iterator it = param["KinematicCuts"][KinVar_i].begin(); + it != param["KinematicCuts"][KinVar_i].end(); ++it) { TempKinematicStrings.push_back(it->first.as()); TempKinematicBounds.push_back(Get2DBounds(it->second)); } - if(TempKinematicStrings.size() == 0) { - MACH3LOG_ERROR("Received a KinematicCuts node but couldn't read the contents (it's a list of single-element dictionaries (python) = map of pairs (C++))"); - MACH3LOG_ERROR("For Param {}", func.name); + if (TempKinematicStrings.size() == 0) { + MACH3LOG_ERROR("Received a KinematicCuts node but couldn't read the " + "contents (it's a list of single-element dictionaries " + "(python) = map of pairs (C++))"); + MACH3LOG_ERROR("For Param {}", func.fancy_name); throw MaCh3Exception(__FILE__, __LINE__); } - }//KinVar_i + } // KinVar_i func.KinematicVarStr = TempKinematicStrings; func.Selection = TempKinematicBounds; } - func.valuePtr = RetPointer(Index); + return func; } // ******************************************** // Get Osc params -OscillationParameter ParameterHandlerGeneric::GetOscillationParameters(const YAML::Node& param, const int Index) { -// ******************************************** - OscillationParameter OscParamInfo; - GetBaseParameter(param, Index, OscParamInfo); - - return OscParamInfo; +OscillationParameter +ParameterHandlerGeneric::GetOscillationParameters(const YAML::Node &) { + // ******************************************** + return OscillationParameter(); } // ******************************************** // HH: Grab the Functional parameters for the relevant SampleName -const std::vector ParameterHandlerGeneric::GetFunctionalParametersFromSampleName(const std::string& SampleName) const { -// ******************************************** - return GetTypeParamsFromSampleName(_fSystToGlobalSystIndexMap[SystType::kFunc], FuncParams, SampleName); +const std::vector +ParameterHandlerGeneric::GetFunctionalParametersFromSampleName( + const std::string &SampleName) const { + // ******************************************** + std::vector returnVec; + for (auto &par : FuncParams) { + if (AppliesToSample(par.index, SampleName)) { + returnVec.push_back(par); + } + } + return returnVec; } // ******************************************** // DB Grab the Normalisation parameters for the relevant SampleName -const std::vector ParameterHandlerGeneric::GetNormParsFromSampleName(const std::string& SampleName) const { -// ******************************************** - return GetTypeParamsFromSampleName(_fSystToGlobalSystIndexMap[SystType::kNorm], NormParams, SampleName); +const std::vector +ParameterHandlerGeneric::GetNormParsFromSampleName( + const std::string &SampleName) const { + // ******************************************** + std::vector returnVec; + for (auto &par : NormParams) { + if (AppliesToSample(par.index, SampleName)) { + returnVec.push_back(par); + } + } + return returnVec; } // ******************************************** // KS Grab the Spline parameters for the relevant SampleName -const std::vector ParameterHandlerGeneric::GetSplineParsFromSampleName(const std::string& SampleName) const { -// ******************************************** - return GetTypeParamsFromSampleName(_fSystToGlobalSystIndexMap[SystType::kSpline], SplineParams, SampleName); -} - -// ******************************************** -template -std::vector ParameterHandlerGeneric::GetTypeParamsFromSampleName(const std::map& indexMap, const std::vector& params, const std::string& SampleName) const { -// ******************************************** - std::vector returnVec; - for (const auto& pair : indexMap) { - const auto& localIndex = pair.first; - const auto& globalIndex = pair.second; - if (AppliesToSample(globalIndex, SampleName)) { - returnVec.push_back(params[localIndex]); +const std::vector +ParameterHandlerGeneric::GetSplineParsFromSampleName( + const std::string &SampleName) const { + // ******************************************** + std::vector returnVec; + for (auto &par : SplineParams) { + if (AppliesToSample(par.index, SampleName)) { + returnVec.push_back(par); } } return returnVec; @@ -375,83 +405,96 @@ std::vector ParameterHandlerGeneric::GetTypeParamsFromSampleName(const s // ******************************************** // DB Grab the number of parameters for the relevant SampleName -int ParameterHandlerGeneric::GetNumParamsFromSampleName(const std::string& SampleName, const SystType Type) { -// ******************************************** +int ParameterHandlerGeneric::GetNumParamsFromSampleName( + const std::string &SampleName, const SystType Type) { + // ******************************************** int returnVal = 0; - IterateOverParams(SampleName, - [&](int i) { return GetParamType(i) == Type; }, // Filter condition - [&](int) { returnVal += 1; } // Action to perform if filter passes + IterateOverParams( + SampleName, + [&](int i) { return GetParamType(i) == Type; }, // Filter condition + [&](int) { returnVal += 1; } // Action to perform if filter passes ); return returnVal; } // ******************************************** // DB Grab the parameter names for the relevant SampleName -const std::vector ParameterHandlerGeneric::GetParsNamesFromSampleName(const std::string& SampleName, const SystType Type) { -// ******************************************** +const std::vector +ParameterHandlerGeneric::GetParsNamesFromSampleName( + const std::string &SampleName, const SystType Type) { + // ******************************************** std::vector returnVec; - IterateOverParams(SampleName, - [&](int i) { return GetParamType(i) == Type; }, // Filter condition - [&](int i) { returnVec.push_back(GetParFancyName(i)); } // Action to perform if filter passes + IterateOverParams( + SampleName, + [&](int i) { return GetParamType(i) == Type; }, // Filter condition + [&](int i) { + returnVec.push_back(GetParFancyName(i)); + } // Action to perform if filter passes ); return returnVec; } // ******************************************** // DB DB Grab the parameter indices for the relevant SampleName -const std::vector ParameterHandlerGeneric::GetParsIndexFromSampleName(const std::string& SampleName, const SystType Type) { -// ******************************************** +const std::vector ParameterHandlerGeneric::GetParsIndexFromSampleName( + const std::string &SampleName, const SystType Type) { + // ******************************************** std::vector returnVec; - IterateOverParams(SampleName, - [&](int i) { return GetParamType(i) == Type; }, // Filter condition - [&](int i) { returnVec.push_back(i); } // Action to perform if filter passes + IterateOverParams( + SampleName, + [&](int i) { return GetParamType(i) == Type; }, // Filter condition + [&](int i) { + returnVec.push_back(i); + } // Action to perform if filter passes ); return returnVec; } // ******************************************** template -void ParameterHandlerGeneric::IterateOverParams(const std::string& SampleName, FilterFunc filter, ActionFunc action) { -// ******************************************** - for (int i = 0; i < _fNumPar; ++i) { - if ((AppliesToSample(i, SampleName)) && filter(i)) { // Common filter logic - action(i); // Specific action for each function +void ParameterHandlerGeneric::IterateOverParams(const std::string &SampleName, + FilterFunc const &filter, + ActionFunc const &action) { + // ******************************************** + for (auto &par : GlobalParams) { + if ((AppliesToSample(par->index, SampleName)) && + filter(par->index)) { // Common filter logic + action(par->index); // Specific action for each function } } } // ******************************************** void ParameterHandlerGeneric::InitParams() { -// ******************************************** - for (int i = 0; i < _fNumPar; ++i) { - //ETA - set the name to be xsec_% as this is what ProcessorMCMC expects - _fNames[i] = "xsec_"+std::to_string(i); + // ******************************************** + for (int i = 0; i < parlist.NumSystematicBasisParameters(); ++i) { + // ETA - set the name to be xsec_% as this is what ProcessorMCMC expects + parlist.params.name[i] = "xsec_" + std::to_string(i); // KS: Plenty - if(_fParamType[i] == kOsc){ - _fNames[i] = _fFancyNames[i]; + if (GetParamType(i) == kOsc) { + parlist.params.name[i] = parlist.params.fancy_name[i]; + + if (GetParam(i)->group != "Osc") { + MACH3LOG_ERROR("Parameter {}, is of type Oscillation but doesn't " + "belong to Osc group", + parlist.params.fancy_name[i]); - if(_ParameterGroup[i] != "Osc"){ - MACH3LOG_ERROR("Parameter {}, is of type Oscillation but doesn't belong to Osc group", _fFancyNames[i]); - MACH3LOG_ERROR("It belongs to {} group", _ParameterGroup[i]); - throw MaCh3Exception(__FILE__ , __LINE__ ); + MACH3LOG_ERROR("It belongs to {} group", GetParam(i)->group); + + throw MaCh3Exception(__FILE__, __LINE__); } } - // Set ParameterHandler parameters (Curr = current, Prop = proposed, Sigma = step) - _fCurrVal[i] = _fPreFitValue[i]; - _fPropVal[i] = _fCurrVal[i]; - } - Randomize(); - //KS: Transfer the starting parameters to the PCA basis, you don't want to start with zero.. - if (pca) { - PCAObj->SetInitialParameters(_fIndivStepScale); + // Set ParameterHandler parameters (Curr = current, Prop = proposed, Sigma = + // step) + SetParCurrProp(i, GetParInit(i)); } } // ******************************************** // Print everything we know about the inputs we're Getting void ParameterHandlerGeneric::Print() { -// ******************************************** + // ******************************************** MACH3LOG_INFO("#################################################"); MACH3LOG_INFO("Printing ParameterHandlerGeneric:"); @@ -475,177 +518,225 @@ void ParameterHandlerGeneric::Print() { // ******************************************** void ParameterHandlerGeneric::PrintGlobablInfo() { -// ******************************************** - MACH3LOG_INFO("============================================================================================================================================================"); - MACH3LOG_INFO("{:<5} {:2} {:<40} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} {:<20} {:2} {:<10}", "#", "|", "Name", "|", "Prior", "|", "Error", "|", "Lower", "|", "Upper", "|", "StepScale", "|", "SampleNames", "|", "Type"); - MACH3LOG_INFO("------------------------------------------------------------------------------------------------------------------------------------------------------------"); - for (int i = 0; i < GetNumParams(); i++) { - std::string ErrString = fmt::format("{:.2f}", _fError[i]); + // ******************************************** + MACH3LOG_INFO("==============================================================" + "==============================================================" + "================================"); + MACH3LOG_INFO("{:<5} {:2} {:<40} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} " + "{:<10} {:2} {:<10} {:2} {:<20} {:2} {:<10}", + "#", "|", "Name", "|", "Prior", "|", "Error", "|", "Lower", "|", + "Upper", "|", "StepScale", "|", "SampleNames", "|", "Type"); + MACH3LOG_INFO("--------------------------------------------------------------" + "--------------------------------------------------------------" + "--------------------------------"); + for (int i = 0; i < GetNumSystematicParams(); i++) { + std::string ErrString = fmt::format("{:.2f}", GetError(i)); std::string SampleNameString = ""; - for (const auto& SampleName : _fSampleNames[i]) { + for (const auto &SampleName : parlist.params.samples[i]) { if (!SampleNameString.empty()) { SampleNameString += ", "; } SampleNameString += SampleName; } - MACH3LOG_INFO("{:<5} {:2} {:<40} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} {:<20} {:2} {:<10}", i, "|", GetParFancyName(i), "|", _fPreFitValue[i], "|", "+/- " + ErrString, "|", _fLowBound[i], "|", _fUpBound[i], "|", _fIndivStepScale[i], "|", SampleNameString, "|", SystType_ToString(_fParamType[i])); + MACH3LOG_INFO("{:<5} {:2} {:<40} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} " + "{:<10} {:2} {:<10} {:2} {:<20} {:2} {:<10}", + i, "|", GetParFancyName(i), "|", GetParInit(i), "|", + "+/- " + ErrString, "|", GetLowerBound(i), "|", + GetUpperBound(i), "|", GetIndivStepScale(i), "|", + SampleNameString, "|", GetParamTypeString(i)); } - MACH3LOG_INFO("============================================================================================================================================================"); + MACH3LOG_INFO("==============================================================" + "==============================================================" + "================================"); } // ******************************************** void ParameterHandlerGeneric::PrintNormParams() { -// ******************************************** + // ******************************************** // Output the normalisation parameters as a sanity check! MACH3LOG_INFO("Normalisation parameters: {}", NormParams.size()); - if(_fSystToGlobalSystIndexMap[SystType::kNorm].size() == 0) return; + if (!NormParams.size()) { + return; + } bool have_parameter_with_kin_bounds = false; - //KS: Consider making some class producing table.. - MACH3LOG_INFO("┌────┬──────────┬────────────────────────────────────────┬────────────────────┬────────────────────┬────────────────────┐"); - MACH3LOG_INFO("│{0:4}│{1:10}│{2:40}│{3:20}│{4:20}│{5:20}│", "#", "Global #", "Name", "Int. mode", "Target", "pdg"); - MACH3LOG_INFO("├────┼──────────┼────────────────────────────────────────┼────────────────────┼────────────────────┼────────────────────┤"); + // KS: Consider making some class producing table.. + MACH3LOG_INFO("┌────┬──────────┬────────────────────────────────────────┬────" + "────────────────┬────────────────────┬────────────────────┐"); + MACH3LOG_INFO("│{0:4}│{1:10}│{2:40}│{3:20}│{4:20}│{5:20}│", "#", "Global #", + "Name", "Int. mode", "Target", "pdg"); + MACH3LOG_INFO("├────┼──────────┼────────────────────────────────────────┼────" + "────────────────┼────────────────────┼────────────────────┤"); - for (unsigned int i = 0; i < NormParams.size(); ++i) - { + for (unsigned int i = 0; i < NormParams.size(); ++i) { std::string intModeString; for (unsigned int j = 0; j < NormParams[i].modes.size(); j++) { intModeString += std::to_string(NormParams[i].modes[j]); intModeString += " "; } - if (NormParams[i].modes.empty()) intModeString += "all"; + if (NormParams[i].modes.empty()) + intModeString += "all"; std::string targetString; for (unsigned int j = 0; j < NormParams[i].targets.size(); j++) { targetString += std::to_string(NormParams[i].targets[j]); targetString += " "; } - if (NormParams[i].targets.empty()) targetString += "all"; + if (NormParams[i].targets.empty()) + targetString += "all"; std::string pdgString; for (unsigned int j = 0; j < NormParams[i].pdgs.size(); j++) { pdgString += std::to_string(NormParams[i].pdgs[j]); pdgString += " "; } - if (NormParams[i].pdgs.empty()) pdgString += "all"; + if (NormParams[i].pdgs.empty()) + pdgString += "all"; - MACH3LOG_INFO("│{: <4}│{: <10}│{: <40}│{: <20}│{: <20}│{: <20}│", i, NormParams[i].index, NormParams[i].name, intModeString, targetString, pdgString); + MACH3LOG_INFO("│{: <4}│{: <10}│{: <40}│{: <20}│{: <20}│{: <20}│", i, + NormParams[i].index, NormParams[i].fancy_name, intModeString, + targetString, pdgString); - if(NormParams[i].hasKinBounds) have_parameter_with_kin_bounds = true; + if (NormParams[i].hasKinBounds) + have_parameter_with_kin_bounds = true; } - MACH3LOG_INFO("└────┴──────────┴────────────────────────────────────────┴────────────────────┴────────────────────┴────────────────────┘"); + MACH3LOG_INFO("└────┴──────────┴────────────────────────────────────────┴────" + "────────────────┴────────────────────┴────────────────────┘"); - if(have_parameter_with_kin_bounds) { + if (have_parameter_with_kin_bounds) { MACH3LOG_INFO("Normalisation parameters KinematicCuts information"); - MACH3LOG_INFO("┌────┬──────────┬────────────────────────────────────────┬────────────────────┬────────────────────────────────────────┐"); - MACH3LOG_INFO("│{0:4}│{1:10}│{2:40}│{3:20}│{4:40}│", "#", "Global #", "Name", "KinematicCut", "Value"); - MACH3LOG_INFO("├────┼──────────┼────────────────────────────────────────┼────────────────────┼────────────────────────────────────────┤"); - for (unsigned int i = 0; i < NormParams.size(); ++i) - { - //skip parameters with no KinematicCuts - if(!NormParams[i].hasKinBounds) continue; + MACH3LOG_INFO( + "┌────┬──────────┬────────────────────────────────────────┬────────────" + "────────┬────────────────────────────────────────┐"); + MACH3LOG_INFO("│{0:4}│{1:10}│{2:40}│{3:20}│{4:40}│", "#", "Global #", + "Name", "KinematicCut", "Value"); + MACH3LOG_INFO( + "├────┼──────────┼────────────────────────────────────────┼────────────" + "────────┼────────────────────────────────────────┤"); + for (unsigned int i = 0; i < NormParams.size(); ++i) { + // skip parameters with no KinematicCuts + if (!NormParams[i].hasKinBounds) + continue; const long unsigned int ncuts = NormParams[i].KinematicVarStr.size(); - for(long unsigned int icut = 0; icut < ncuts; icut++) { + for (long unsigned int icut = 0; icut < ncuts; icut++) { std::string kinematicCutValueString; - for(const auto & value : NormParams[i].Selection[icut]) { - for (const auto& v : value) { + for (const auto &value : NormParams[i].Selection[icut]) { + for (const auto &v : value) { kinematicCutValueString += fmt::format("{:.2f} ", v); } } - if(icut == 0) - MACH3LOG_INFO("│{: <4}│{: <10}│{: <40}│{: <20}│{: <40}│", i, NormParams[i].index, NormParams[i].name, NormParams[i].KinematicVarStr[icut], kinematicCutValueString); + if (icut == 0) + MACH3LOG_INFO("│{: <4}│{: <10}│{: <40}│{: <20}│{: <40}│", i, + NormParams[i].index, NormParams[i].fancy_name, + NormParams[i].KinematicVarStr[icut], + kinematicCutValueString); else - MACH3LOG_INFO("│{: <4}│{: <10}│{: <40}│{: <20}│{: <40}│", "", "", "", NormParams[i].KinematicVarStr[icut], kinematicCutValueString); - }//icut - }//i - MACH3LOG_INFO("└────┴──────────┴────────────────────────────────────────┴────────────────────┴────────────────────────────────────────┘"); - } - else + MACH3LOG_INFO("│{: <4}│{: <10}│{: <40}│{: <20}│{: <40}│", "", "", "", + NormParams[i].KinematicVarStr[icut], + kinematicCutValueString); + } // icut + } // i + MACH3LOG_INFO( + "└────┴──────────┴────────────────────────────────────────┴────────────" + "────────┴────────────────────────────────────────┘"); + } else MACH3LOG_INFO("No normalisation parameters have KinematicCuts defined"); } // ******************************************** void ParameterHandlerGeneric::PrintSplineParams() { -// ******************************************** - MACH3LOG_INFO("Spline parameters: {}", _fSystToGlobalSystIndexMap[SystType::kSpline].size()); - if(_fSystToGlobalSystIndexMap[SystType::kSpline].size() == 0) return; - MACH3LOG_INFO("====================================================================================================================================================================="); - MACH3LOG_INFO("{:<4} {:<2} {:<40} {:<2} {:<40} {:<2} {:<20} {:<2} {:<20} {:<2} {:<20} {:<2}", "#", "|", "Name", "|", "Spline Name", "|", "Spline Interpolation", "|", "Low Knot Bound", "|", "Up Knot Bound", "|"); - MACH3LOG_INFO("---------------------------------------------------------------------------------------------------------------------------------------------------------------------"); - for (auto &pair : _fSystToGlobalSystIndexMap[SystType::kSpline]) { - auto &SplineIndex = pair.first; - auto &GlobalIndex = pair.second; - - MACH3LOG_INFO("{:<4} {:<2} {:<40} {:<2} {:<40} {:<2} {:<20} {:<2} {:<20} {:<2} {:<20} {:<2}", - SplineIndex, "|", GetParFancyName(GlobalIndex), "|", - _fSplineNames[SplineIndex], "|", - SplineInterpolation_ToString(GetParSplineInterpolation(SplineIndex)), "|", - GetParSplineKnotLowerBound(SplineIndex), "|", - GetParSplineKnotUpperBound(SplineIndex), "|"); + // ******************************************** + MACH3LOG_INFO("Spline parameters: {}", SplineParams.size()); + if (SplineParams.size() == 0) { + return; + } + MACH3LOG_INFO("==============================================================" + "==============================================================" + "========================================="); + MACH3LOG_INFO("{:<4} {:<2} {:<40} {:<2} {:<40} {:<2} {:<20} {:<2} {:<20} " + "{:<2} {:<20} {:<2}", + "#", "|", "Name", "|", "Spline Name", "|", + "Spline Interpolation", "|", "Low Knot Bound", "|", + "Up Knot Bound", "|"); + MACH3LOG_INFO("--------------------------------------------------------------" + "--------------------------------------------------------------" + "-----------------------------------------"); + for (int i = 0; i < int(SplineParams.size()); ++i) { + MACH3LOG_INFO( + "{:<4} {:<2} {:<40} {:<2} {:<40} {:<2} {:<20} {:<2} {:<20} {:<2} " + "{:<20} {:<2}", + i, "|", GetParFancyName(SplineParams[i].index), "|", + SplineParams[i].spline_name, "|", + SplineInterpolation_ToString(GetParSplineInterpolation(i)), "|", + GetParSplineKnotLowerBound(i), "|", GetParSplineKnotUpperBound(i), "|"); } - MACH3LOG_INFO("====================================================================================================================================================================="); + MACH3LOG_INFO("==============================================================" + "==============================================================" + "========================================="); } // ******************************************** void ParameterHandlerGeneric::PrintFunctionalParams() { -// ******************************************** - MACH3LOG_INFO("Functional parameters: {}", _fSystToGlobalSystIndexMap[SystType::kFunc].size()); - if(_fSystToGlobalSystIndexMap[SystType::kFunc].size() == 0) return; + // ******************************************** + MACH3LOG_INFO("Functional parameters: {}", FuncParams.size()); + if (FuncParams.size() == 0) { + return; + } MACH3LOG_INFO("┌────┬──────────┬────────────────────────────────────────┐"); MACH3LOG_INFO("│{0:4}│{1:10}│{2:40}│", "#", "Global #", "Name"); MACH3LOG_INFO("├────┼──────────┼────────────────────────────────────────┤"); - for (auto &pair : _fSystToGlobalSystIndexMap[SystType::kFunc]) { - auto &FuncIndex = pair.first; - auto &GlobalIndex = pair.second; - MACH3LOG_INFO("│{0:4}│{1:<10}│{2:40}│", std::to_string(FuncIndex), GlobalIndex, GetParFancyName(GlobalIndex)); + for (int i = 0; i < int(FuncParams.size()); ++i) { + MACH3LOG_INFO("│{0:4}│{1:<10}│{2:40}│", std::to_string(i), + FuncParams[i].index, GetParFancyName(FuncParams[i].index)); } MACH3LOG_INFO("└────┴──────────┴────────────────────────────────────────┘"); } // ******************************************** void ParameterHandlerGeneric::PrintOscillationParams() { -// ******************************************** - MACH3LOG_INFO("Oscillation parameters: {}", _fSystToGlobalSystIndexMap[SystType::kOsc].size()); - if(_fSystToGlobalSystIndexMap[SystType::kOsc].size() == 0) return; + // ******************************************** + MACH3LOG_INFO("Oscillation parameters: {}", OscParams.size()); + if (OscParams.size() == 0) { + return; + } MACH3LOG_INFO("┌────┬──────────┬────────────────────────────────────────┐"); MACH3LOG_INFO("│{0:4}│{1:10}│{2:40}│", "#", "Global #", "Name"); MACH3LOG_INFO("├────┼──────────┼────────────────────────────────────────┤"); - for (auto &pair : _fSystToGlobalSystIndexMap[SystType::kOsc]) { - auto &OscIndex = pair.first; - auto &GlobalIndex = pair.second; - MACH3LOG_INFO("│{0:4}│{1:<10}│{2:40}│", std::to_string(OscIndex), GlobalIndex, GetParFancyName(GlobalIndex)); + for (int i = 0; i < int(OscParams.size()); ++i) { + MACH3LOG_INFO("│{0:4}│{1:<10}│{2:40}│", std::to_string(i), + OscParams[i].index, GetParFancyName(OscParams[i].index)); } MACH3LOG_INFO("└────┴──────────┴────────────────────────────────────────┘"); } // ******************************************** void ParameterHandlerGeneric::PrintParameterGroups() { -// ******************************************** - // KS: Create a map to store the counts of unique strings, in principle this could be in header file + // ******************************************** + // KS: Create a map to store the counts of unique strings, in principle this + // could be in header file std::unordered_map paramCounts; - std::for_each(_ParameterGroup.begin(), _ParameterGroup.end(), - [¶mCounts](const std::string& param) { - paramCounts[param]++; - }); + for (auto &par : GlobalParams) { + paramCounts[par->group]++; + } MACH3LOG_INFO("Printing parameter groups"); // Output the counts - for (const auto& pair : paramCounts) { + for (const auto &pair : paramCounts) { MACH3LOG_INFO("Found {}: {} params", pair.second, pair.first); } } // ******************************************** std::vector ParameterHandlerGeneric::GetUniqueParameterGroups() { -// ******************************************** + // ******************************************** std::unordered_set uniqueGroups; // Fill the set with unique values - for (const auto& param : _ParameterGroup) { - uniqueGroups.insert(param); + for (auto &par : GlobalParams) { + uniqueGroups.insert(par->group); } // Convert to vector and return @@ -656,208 +747,239 @@ std::vector ParameterHandlerGeneric::GetUniqueParameterGroups() { // ******************************************** // KS: Check if matrix is correctly initialised void ParameterHandlerGeneric::CheckCorrectInitialisation() { -// ******************************************** - // KS: Lambda Function which simply checks if there are no duplicates in std::vector - auto CheckForDuplicates = [](const std::vector& names, const std::string& nameType) { + // ******************************************** + // KS: Lambda Function which simply checks if there are no duplicates in + // std::vector + auto CheckForDuplicates = [](const std::vector &names, + const std::string &nameType) { std::unordered_map seenStrings; for (size_t i = 0; i < names.size(); ++i) { - const auto& name = names[i]; + const auto &name = names[i]; if (seenStrings.find(name) != seenStrings.end()) { size_t firstIndex = seenStrings[name]; - MACH3LOG_CRITICAL("There are two systematics with the same {} '{}', first at index {}, and again at index {}", nameType, name, firstIndex, i); + MACH3LOG_CRITICAL("There are two systematics with the same {} '{}', " + "first at index {}, and again at index {}", + nameType, name, firstIndex, i); throw MaCh3Exception(__FILE__, __LINE__); } seenStrings[name] = i; } }; - // KS: Checks if there are no duplicates in fancy names etc, this can happen if we merge configs etc - CheckForDuplicates(_fFancyNames, "_fFancyNames"); - CheckForDuplicates(_fSplineNames, "_fSplineNames"); + // KS: Checks if there are no duplicates in fancy names etc, this can happen + // if we merge configs etc + CheckForDuplicates(parlist.params.fancy_name, "fancy_name"); + + std::vector spline_names; + for (auto &sp : SplineParams) { + spline_names.push_back(sp.spline_name); + } + CheckForDuplicates(spline_names, "spline_names"); } // ******************************************** // Function to set to prior parameters of a given group -void ParameterHandlerGeneric::SetGroupOnlyParameters(const std::vector< std::string>& Groups) { -// ******************************************** - for(size_t i = 0; i < Groups.size(); i++){ +void ParameterHandlerGeneric::SetGroupOnlyParameters( + const std::vector &Groups) { + // ******************************************** + for (size_t i = 0; i < Groups.size(); i++) { SetGroupOnlyParameters(Groups[i]); } } // ******************************************** // Function to set to prior parameters of a given group -void ParameterHandlerGeneric::SetGroupOnlyParameters(const std::string& Group, const std::vector& Pars) { -// ******************************************** +void ParameterHandlerGeneric::SetGroupOnlyParameters( + const std::string &Group, const std::vector &Pars) { + // ******************************************** // If empty, set the proposed to prior if (Pars.empty()) { - for (int i = 0; i < _fNumPar; i++) { - if(IsParFromGroup(i, Group)) _fPropVal[i] = _fPreFitValue[i]; + for (int i = 0; i < parlist.NumSystematicBasisParameters(); i++) { + if (IsParFromGroup(i, Group)) { + SetParProp(i, GetParInit(i)); + } } - } else{ + } else { const size_t ExpectedSize = static_cast(GetNumParFromGroup(Group)); if (Pars.size() != ExpectedSize) { - MACH3LOG_ERROR("Number of param in group {} is {}, while you passed {}", Group, ExpectedSize, Pars.size()); + MACH3LOG_ERROR("Number of param in group {} is {}, while you passed {}", + Group, ExpectedSize, Pars.size()); throw MaCh3Exception(__FILE__, __LINE__); } int Counter = 0; - for (int i = 0; i < _fNumPar; i++) { - // If belongs to group set value from parsed vector, otherwise use propose value - if(IsParFromGroup(i, Group)){ - _fPropVal[i] = Pars[Counter]; + for (int i = 0; i < parlist.NumSystematicBasisParameters(); i++) { + // If belongs to group set value from parsed vector, otherwise use propose + // value + if (IsParFromGroup(i, Group)) { + SetParProp(i, Pars[Counter]); Counter++; } } } - // And if pca make the transfer - if (pca) { - PCAObj->TransferToPCA(); - PCAObj->TransferToParam(); - } } // ******************************************** // Toggle fix/free to parameters of a given group -void ParameterHandlerGeneric::ToggleFixGroupOnlyParameters(const std::string& Group) { -// ******************************************** - for (int i = 0; i < _fNumPar; ++i) - if(IsParFromGroup(i, Group)) ToggleFixParameter(i); +void ParameterHandlerGeneric::ToggleFixGroupOnlyParameters( + const std::string &Group) { + // ******************************************** + for (int i = 0; i < parlist.NumSystematicBasisParameters(); ++i) + if (IsParFromGroup(i, Group)) + ToggleFixParameter(i); } // ******************************************** // Toggle fix/free to parameters of several groups -void ParameterHandlerGeneric::ToggleFixGroupOnlyParameters(const std::vector< std::string>& Groups) { -// ******************************************** - for(size_t i = 0; i < Groups.size(); i++) - ToggleFixGroupOnlyParameters(Groups[i]); +void ParameterHandlerGeneric::ToggleFixGroupOnlyParameters( + const std::vector &Groups) { + // ******************************************** + for (size_t i = 0; i < Groups.size(); i++) + ToggleFixGroupOnlyParameters(Groups[i]); } // ******************************************** // Set parameters to be fixed in a given group -void ParameterHandlerGeneric::SetFixGroupOnlyParameters(const std::string& Group) { -// ******************************************** - for (int i = 0; i < _fNumPar; ++i) - if(IsParFromGroup(i, Group)) SetFixParameter(i); +void ParameterHandlerGeneric::SetFixGroupOnlyParameters( + const std::string &Group) { + // ******************************************** + for (int i = 0; i < parlist.NumSystematicBasisParameters(); ++i) + if (IsParFromGroup(i, Group)) + SetFixParameter(i); } // ******************************************** // Set parameters of several groups to be fixed -void ParameterHandlerGeneric::SetFixGroupOnlyParameters(const std::vector< std::string>& Groups) { -// ******************************************** - for(size_t i = 0; i < Groups.size(); i++) - SetFixGroupOnlyParameters(Groups[i]); +void ParameterHandlerGeneric::SetFixGroupOnlyParameters( + const std::vector &Groups) { + // ******************************************** + for (size_t i = 0; i < Groups.size(); i++) + SetFixGroupOnlyParameters(Groups[i]); } // ******************************************** // Set parameters to be free in a given group -void ParameterHandlerGeneric::SetFreeGroupOnlyParameters(const std::string& Group) { -// ******************************************** - for (int i = 0; i < _fNumPar; ++i) - if(IsParFromGroup(i, Group)) SetFreeParameter(i); +void ParameterHandlerGeneric::SetFreeGroupOnlyParameters( + const std::string &Group) { + // ******************************************** + for (int i = 0; i < parlist.NumSystematicBasisParameters(); ++i) + if (IsParFromGroup(i, Group)) + SetFreeParameter(i); } // ******************************************** // Set parameters of several groups to be fixed -void ParameterHandlerGeneric::SetFreeGroupOnlyParameters(const std::vector< std::string>& Groups) { -// ******************************************** - for(size_t i = 0; i < Groups.size(); i++) - SetFreeGroupOnlyParameters(Groups[i]); +void ParameterHandlerGeneric::SetFreeGroupOnlyParameters( + const std::vector &Groups) { + // ******************************************** + for (size_t i = 0; i < Groups.size(); i++) + SetFreeGroupOnlyParameters(Groups[i]); } // ******************************************** // Checks if parameter belongs to a given group -bool ParameterHandlerGeneric::IsParFromGroup(const int i, const std::string& Group) const { -// ******************************************** +bool ParameterHandlerGeneric::IsParFromGroup(const int i, + const std::string &Group) const { + // ******************************************** std::string groupLower = Group; - std::string paramGroupLower = _ParameterGroup[i]; - - // KS: Convert both strings to lowercase, this way comparison will be case insensitive - std::transform(groupLower.begin(), groupLower.end(), groupLower.begin(), ::tolower); - std::transform(paramGroupLower.begin(), paramGroupLower.end(), paramGroupLower.begin(), ::tolower); + auto par = GetParam(i); + std::string paramGroupLower = par ? par->group : std::string("unmanaged"); + // KS: Convert both strings to lowercase, this way comparison will be case + // insensitive + std::transform(groupLower.begin(), groupLower.end(), groupLower.begin(), + ::tolower); + std::transform(paramGroupLower.begin(), paramGroupLower.end(), + paramGroupLower.begin(), ::tolower); return groupLower == paramGroupLower; } // ******************************************** -int ParameterHandlerGeneric::GetNumParFromGroup(const std::string& Group) const { -// ******************************************** +int ParameterHandlerGeneric::GetNumParFromGroup( + const std::string &Group) const { + // ******************************************** int Counter = 0; - for (int i = 0; i < _fNumPar; i++) { - if(IsParFromGroup(i, Group)) Counter++; + for (int i = 0; i < parlist.NumSystematicBasisParameters(); i++) { + if (IsParFromGroup(i, Group)) + Counter++; } return Counter; } // ******************************************** // DB Grab the Normalisation parameters for the relevant sample name -std::vector ParameterHandlerGeneric::GetOscParsFromSampleName(const std::string& SampleName) { -// ******************************************** - std::vector returnVec; - for (const auto& pair : _fSystToGlobalSystIndexMap[SystType::kOsc]) { - const auto& globalIndex = pair.second; - if (AppliesToSample(globalIndex, SampleName)) { - returnVec.push_back(RetPointer(globalIndex)); +std::vector ParameterHandlerGeneric::GetOscParsFromSampleName( + const std::string &SampleName) { + // ******************************************** + std::vector returnVec; + for (auto &par : OscParams) { + if (AppliesToSample(par.index, SampleName)) { + returnVec.push_back(par.valuePtr); } } return returnVec; } // ******************************************** -// Dump Matrix to ROOT file, useful when we need to pass matrix info to another fitting group -void ParameterHandlerGeneric::DumpMatrixToFile(const std::string& Name) { -// ******************************************** - TFile* outputFile = new TFile(Name.c_str(), "RECREATE"); - - TObjArray* xsec_param_names = new TObjArray(); - TObjArray* xsec_spline_interpolation = new TObjArray(); - TObjArray* xsec_spline_names = new TObjArray(); - - TVectorD* xsec_param_prior = new TVectorD(_fNumPar); - TVectorD* xsec_flat_prior = new TVectorD(_fNumPar); - TVectorD* xsec_stepscale = new TVectorD(_fNumPar); - TVectorD* xsec_param_lb = new TVectorD(_fNumPar); - TVectorD* xsec_param_ub = new TVectorD(_fNumPar); - - TVectorD* xsec_param_knot_weight_lb = new TVectorD(_fNumPar); - TVectorD* xsec_param_knot_weight_ub = new TVectorD(_fNumPar); - TVectorD* xsec_error = new TVectorD(_fNumPar); - - for(int i = 0; i < _fNumPar; ++i) - { - TObjString* nameObj = new TObjString(_fFancyNames[i].c_str()); +// Dump Matrix to ROOT file, useful when we need to pass matrix info to another +// fitting group +void ParameterHandlerGeneric::DumpMatrixToFile(const std::string &Name) { + // ******************************************** + TFile *outputFile = new TFile(Name.c_str(), "RECREATE"); + + TObjArray *xsec_param_names = new TObjArray(); + TObjArray *xsec_spline_interpolation = new TObjArray(); + TObjArray *xsec_spline_names = new TObjArray(); + + TVectorD *xsec_param_prior = + new TVectorD(parlist.NumSystematicBasisParameters()); + TVectorD *xsec_flat_prior = + new TVectorD(parlist.NumSystematicBasisParameters()); + TVectorD *xsec_stepscale = + new TVectorD(parlist.NumSystematicBasisParameters()); + TVectorD *xsec_param_lb = + new TVectorD(parlist.NumSystematicBasisParameters()); + TVectorD *xsec_param_ub = + new TVectorD(parlist.NumSystematicBasisParameters()); + + TVectorD *xsec_param_knot_weight_lb = + new TVectorD(parlist.NumSystematicBasisParameters()); + TVectorD *xsec_param_knot_weight_ub = + new TVectorD(parlist.NumSystematicBasisParameters()); + TVectorD *xsec_error = new TVectorD(parlist.NumSystematicBasisParameters()); + + for (int i = 0; i < parlist.NumSystematicBasisParameters(); ++i) { + TObjString *nameObj = new TObjString(GetParFancyName(i).c_str()); xsec_param_names->AddLast(nameObj); - TObjString* splineType = new TObjString("TSpline3"); + TObjString *splineType = new TObjString("TSpline3"); xsec_spline_interpolation->AddLast(splineType); - TObjString* splineName = new TObjString(""); + TObjString *splineName = new TObjString(""); xsec_spline_names->AddLast(splineName); - (*xsec_param_prior)[i] = _fPreFitValue[i]; - (*xsec_flat_prior)[i] = _fFlatPrior[i]; - (*xsec_stepscale)[i] = _fIndivStepScale[i]; - (*xsec_error)[i] = _fError[i]; + (*xsec_param_prior)[i] = GetParInit(i); + (*xsec_flat_prior)[i] = GetFlatPrior(i); + (*xsec_stepscale)[i] = GetIndivStepScale(i); + (*xsec_error)[i] = GetError(i); - (*xsec_param_lb)[i] = _fLowBound[i]; - (*xsec_param_ub)[i] = _fUpBound[i]; + (*xsec_param_lb)[i] = GetLowerBound(i); + (*xsec_param_ub)[i] = GetUpperBound(i); //Default values (*xsec_param_knot_weight_lb)[i] = -9999; (*xsec_param_knot_weight_ub)[i] = +9999; } - for (auto &pair : _fSystToGlobalSystIndexMap[SystType::kSpline]) { - auto &SplineIndex = pair.first; - auto &SystIndex = pair.second; + for (auto &sp : SplineParams) { - (*xsec_param_knot_weight_lb)[SystIndex] = SplineParams.at(SplineIndex)._SplineKnotLowBound; - (*xsec_param_knot_weight_ub)[SystIndex] = SplineParams.at(SplineIndex)._SplineKnotUpBound; + (*xsec_param_knot_weight_lb)[sp.index] = sp.SplineKnotLowBound; + (*xsec_param_knot_weight_ub)[sp.index] = sp.SplineKnotUpBound; - TObjString* splineType = new TObjString(SplineInterpolation_ToString(SplineParams.at(SplineIndex)._SplineInterpolationType).c_str()); - xsec_spline_interpolation->AddAt(splineType, SystIndex); + TObjString* splineType = new TObjString(SplineInterpolation_ToString(sp.SplineInterpolationType).c_str()); + xsec_spline_interpolation->AddAt(splineType, sp.index); - TObjString* splineName = new TObjString(_fSplineNames[SplineIndex].c_str()); - xsec_spline_names->AddAt(splineName, SystIndex); + TObjString* splineName = new TObjString(sp.spline_name.c_str()); + xsec_spline_names->AddAt(splineName, sp.index); } xsec_param_names->Write("xsec_param_names", TObject::kSingleKey); delete xsec_param_names; diff --git a/Parameters/ParameterHandlerGeneric.h b/Parameters/ParameterHandlerGeneric.h index 45b074731..728e0e566 100644 --- a/Parameters/ParameterHandlerGeneric.h +++ b/Parameters/ParameterHandlerGeneric.h @@ -4,253 +4,291 @@ #include "Parameters/ParameterHandlerBase.h" #include "Samples/SampleStructs.h" -/// @brief Class responsible for handling of systematic error parameters with different types defined in the config. Like spline, normalisation parameters etc. -/// @see For more details, visit the [Wiki](https://github.com/mach3-software/MaCh3/wiki/02.-Implementation-of-Systematic). +/// @brief Class responsible for handling of systematic error parameters with +/// different types defined in the config. Like spline, normalisation parameters +/// etc. +/// @see For more details, visit the +/// [Wiki](https://github.com/mach3-software/MaCh3/wiki/02.-Implementation-of-Systematic). /// @author Dan Barrow /// @author Ed Atkin /// @author Kamil Skwarczynski class ParameterHandlerGeneric : public ParameterHandlerBase { - public: - /// @brief Constructor - /// @param FileNames A vector of strings representing the YAML files used for initialisation of matrix - /// @param name Matrix name - /// @param threshold PCA threshold from 0 to 1. Default is -1 and means no PCA - /// @param FirstPCAdpar First PCA parameter that will be decomposed. - /// @param LastPCAdpar First PCA parameter that will be decomposed. - ParameterHandlerGeneric(const std::vector& FileNames, std::string name = "xsec_cov", double threshold = -1, int FirstPCAdpar = -999, int LastPCAdpar = -999); - /// @brief Destructor - ~ParameterHandlerGeneric(); - - // General Getter functions not split by detector - /// @brief ETA - just return the int of the SampleName, this can be removed to do a string comp at some point. - /// @param i parameter index - /// @ingroup ParameterHandlerGetters - inline std::vector GetParSampleID(const int i) const { return _fSampleNames[i];}; - /// @brief ETA - just return a string of "spline", "norm" or "functional" - /// @param i parameter index - /// @ingroup ParameterHandlerGetters - inline std::string GetParamTypeString(const int i) const { return SystType_ToString(_fParamType[i]); } - /// @brief Returns enum describing our param type - /// @param i parameter index - /// @ingroup ParameterHandlerGetters - inline SystType GetParamType(const int i) const {return _fParamType[i];} - - /// @brief Get interpolation type for a given parameter - /// @param i spline parameter index, not confuse with global index - inline SplineInterpolation GetParSplineInterpolation(const int i) const {return SplineParams.at(i)._SplineInterpolationType;} - /// @brief Get the interpolation types for splines affecting a particular SampleName - /// @ingroup ParameterHandlerGetters - const std::vector GetSplineInterpolationFromSampleName(const std::string& SampleName); - /// @brief Get the name of the spline associated with the spline at index i - /// @param i spline parameter index, not to be confused with global index - /// @ingroup ParameterHandlerGetters - std::string GetParSplineName(const int i) const {return _fSplineNames[i];} - - /// @brief DB Get spline parameters depending on given SampleName - /// @ingroup ParameterHandlerGetters - const std::vector GetGlobalSystIndexFromSampleName(const std::string& SampleName, const SystType Type); - /// @brief EM: value at which we cap spline knot weight - /// @param i spline parameter index, not confuse with global index - /// @ingroup ParameterHandlerGetters - inline double GetParSplineKnotUpperBound(const int i) const {return SplineParams.at(i)._SplineKnotUpBound;} - /// @brief EM: value at which we cap spline knot weight - /// @param i spline parameter index, not confuse with global index - /// @ingroup ParameterHandlerGetters - inline double GetParSplineKnotLowerBound(const int i) const {return SplineParams.at(i)._SplineKnotLowBound;} - - /// @brief DB Grab the number of parameters for the relevant SampleName - /// @param SampleName property of SampleHandler class based on which we select whether to apply uncertainties or not - /// @param Type Type of syst, for example kNorm, kSpline etc - /// @ingroup ParameterHandlerGetters - int GetNumParamsFromSampleName(const std::string& SampleName, const SystType Type); - /// @brief DB Grab the parameter names for the relevant SampleName - /// @param SampleName property of SampleHandler class based on which we select whether to apply uncertainties or not - /// @param Type Type of syst, for example kNorm, kSpline etc - /// @ingroup ParameterHandlerGetters - const std::vector GetParsNamesFromSampleName(const std::string& SampleName, const SystType Type); - /// @brief DB Grab the parameter indices for the relevant SampleName - /// @param SampleName property of SampleHandler class based on which we select whether to apply uncertainties or not - /// @param Type Type of syst, for example kNorm, kSpline etc - /// @ingroup ParameterHandlerGetters - const std::vector GetParsIndexFromSampleName(const std::string& SampleName, const SystType Type); - - /// @brief DB Get spline parameters depending on given SampleName - /// @ingroup ParameterHandlerGetters - const std::vector GetSplineParsNamesFromSampleName(const std::string& SampleName); - /// @brief DB Get spline parameters depending on given SampleName - /// @ingroup ParameterHandlerGetters - const std::vector GetSplineFileParsNamesFromSampleName(const std::string& SampleName); - - /// @brief DB Grab the Spline Modes for the relevant SampleName - /// @ingroup ParameterHandlerGetters - const std::vector< std::vector > GetSplineModeVecFromSampleName(const std::string& SampleName); - /// @brief Grab the index of the syst relative to global numbering. - /// @param SampleName property of SampleHandler class based on which we select whether to apply uncertainties or not - /// @param Type Type of syst, for example kNorm, kSpline etc - /// @ingroup ParameterHandlerGetters - const std::vector GetSystIndexFromSampleName(const std::string& SampleName, const SystType Type) const; - /// @brief DB Get norm/func parameters depending on given SampleName - /// @ingroup ParameterHandlerGetters - const std::vector GetNormParsFromSampleName(const std::string& SampleName) const; - /// @brief HH Get functional parameters for the relevant SampleName - /// @ingroup ParameterHandlerGetters - const std::vector GetFunctionalParametersFromSampleName(const std::string& SampleName) const; - /// @brief KS: Grab the Spline parameters for the relevant SampleName - /// @ingroup ParameterHandlerGetters - const std::vector GetSplineParsFromSampleName(const std::string& SampleName) const; - - /// @brief Checks if parameter belongs to a given group - /// @param i parameter index - /// @param Group name of group, like Xsec or Flux - /// @return bool telling whether param is part of group - /// @ingroup ParameterHandlerGetters - bool IsParFromGroup(const int i, const std::string& Group) const; - - /// @brief KS: Check how many parameters are associated with given group - /// @ingroup ParameterHandlerGetters - int GetNumParFromGroup(const std::string& Group) const; - /// @brief KS: Get names of all unique parameter groups - /// @ingroup ParameterHandlerGetters - std::vector GetUniqueParameterGroups(); - - /// @brief KS Function to set to prior parameters of a given group or values from vector - /// @param Group name of group, like Xsec or Flux - /// @param Pars Values which will overwrite proposed step - /// @ingroup ParameterHandlerSetters - /// @note this mimic functionality of @ParameterHandlerBase::SetParameters - void SetGroupOnlyParameters(const std::string& Group, const std::vector& Pars = {}); - /// @brief KS Function to set to prior parameters of a given groups or values from vector - /// @param Groups vector of group names, like Xsec or Flux - /// @ingroup ParameterHandlerSetters - void SetGroupOnlyParameters(const std::vector& Groups); - - /// @brief TN Method to set parameters within a group to be fixed to their prior values - /// @param Group name of the parameter group (Xsec, Flux, Osc, etc.) - void SetFixGroupOnlyParameters(const std::string& Group); - /// @brief TN Method to set parameters of certain groups to be fixed to their prior values - /// @param Groups vector of group names (e.g. {"Xsec", "Flux"}) - void SetFixGroupOnlyParameters(const std::vector& Groups); - - /// @brief TN Method to set parameters within a group to be treated as free - /// @param Group name of the parameter group (Xsec, Flux, Osc, etc.) - void SetFreeGroupOnlyParameters(const std::string& Group); - /// @brief TN Method to set parameters of certain groups to be treated as free - /// @param Groups vector of group names (e.g. {"Xsec", "Flux"}) - void SetFreeGroupOnlyParameters(const std::vector& Groups); - - /// @brief TN Method to toggle fix/free parameters within a group - /// @param Group name of the parameter group (Xsec, Flux, Osc, etc.) - void ToggleFixGroupOnlyParameters(const std::string& Group); - /// @brief TN Method to toggle fix/free parameters within given groups - /// @param Group vector of group names (e.g. {"Xsec", "Flux"}) - void ToggleFixGroupOnlyParameters(const std::vector& Groups); - - /// @brief Dump Matrix to ROOT file, useful when we need to pass matrix info to another fitting group - /// @param Name Name of TFile to which we save stuff - /// @warning This is mostly used for backward compatibility - void DumpMatrixToFile(const std::string& Name); - - /// @brief Get pointers to Osc params from Sample name - /// @ingroup ParameterHandlerGetters - std::vector GetOscParsFromSampleName(const std::string& SampleName); - - protected: - /// @brief Print information about the whole object once it is set - void Print(); - /// @brief Prints general information about the ParameterHandler object. - void PrintGlobablInfo(); - /// @brief Prints normalization parameters. - void PrintNormParams(); - /// @brief Prints spline parameters. - void PrintSplineParams(); - /// @brief Prints functional parameters. - void PrintFunctionalParams(); - /// @brief Prints oscillation parameters. - void PrintOscillationParams(); - /// @brief Prints groups of parameters. - void PrintParameterGroups(); - - /// @brief KS: Check if matrix is correctly initialised - void CheckCorrectInitialisation(); - - /// @brief Iterates over parameters and applies a filter and action function. - /// - /// This template function provides a way to iterate over parameters associated - /// with a specific Sample ID (SampleName). It applies a filter function to determine - /// which parameters to process and an action function to define what to do - /// with the selected parameters. - /// - /// @tparam FilterFunc The type of the filter function used to determine - /// which parameters to include. - /// @tparam ActionFunc The type of the action function applied to each selected - /// parameter. - /// @param SampleName The Sample ID used to filter parameters. - template - void IterateOverParams(const std::string& SampleName, FilterFunc filter, ActionFunc action); - - /// @brief Initializes the systematic parameters from the configuration file. - /// This function loads parameters like normalizations and splines from the provided YAML file. - /// @note This is used internally during the object's initialization process. - void InitParams(); - - /// @brief Parses the YAML configuration to set up cross-section parameters. - /// The YAML file defines the types of systematic errors, interpolation types, and bounds for splines. - inline void InitParametersTypeFromConfig(); - - /// @brief Get Norm params - /// @param param Yaml node describing param - /// @param Index Global parameter index - inline NormParameter GetNormParameter(const YAML::Node& param, const int Index); - - /// @brief Get Osc params - /// @param param Yaml node describing param - /// @param Index Global parameter index - inline OscillationParameter GetOscillationParameters(const YAML::Node& param, const int Index); - - /// @brief Get Func params - /// @param param Yaml node describing param - /// @param Index Global parameter index - inline FunctionalParameter GetFunctionalParameters(const YAML::Node& param, const int Index); - /// @brief Get Spline params - /// @param param Yaml node describing param - /// @param Index Global parameter index - inline SplineParameter GetSplineParameter(const YAML::Node& param, const int Index); - /// @brief Fill base parameters - /// @param param Yaml node describing param - /// @param Index Global parameter index - /// @param Parameter Object storing info - inline void GetBaseParameter(const YAML::Node& param, const int Index, TypeParameterBase& Parameter); - - /// @brief Retrieve parameters that apply to a given sample name. - /// @tparam ParamT Type of parameter (e.g., FunctionalParameter, NormParameter). - /// @param indexMap Map from local to global parameter indices. - /// @param params Vector of all parameters of the given type. - /// @param SampleName The name of the sample to filter applicable parameters for. - /// @return Vector of parameters of type ParamT that apply to the specified sample. - template - std::vector GetTypeParamsFromSampleName(const std::map& indexMap, const std::vector& params, const std::string& SampleName) const; - - /// Type of parameter like norm, spline etc. - std::vector _fParamType; - - /// Name of spline in TTree (TBranch), - std::vector _fSplineNames; - - /// KS: Allow to group parameters for example to affect only cross-section or only flux etc. - std::vector _ParameterGroup; - - /// Map between number of given parameter type with global parameter numbering. For example 2nd norm param may be 10-th global param - std::vector> _fSystToGlobalSystIndexMap; - - /// Vector containing info for normalisation systematics - std::vector SplineParams; - - /// Vector containing info for normalisation systematics - std::vector NormParams; - - /// Vector containing info for functional systematics - std::vector FuncParams; - - /// Vector containing info for functional systematics - std::vector OscParams; +public: + /// @brief Constructor + /// @param FileNames A vector of strings representing the YAML files used for + /// initialisation of matrix + /// @param name Matrix name + /// @param threshold PCA threshold from 0 to 1. Default is -1 and means no PCA + /// @param FirstPCAdpar First PCA parameter that will be decomposed. + /// @param LastPCAdpar First PCA parameter that will be decomposed. + ParameterHandlerGeneric(const std::vector &FileNames, + std::string name = "xsec_cov", double threshold = -1, + int FirstPCAdpar = -999, int LastPCAdpar = -999); + /// @brief Destructor + ~ParameterHandlerGeneric(); + + // General Getter functions not split by detector + /// @brief ETA - just return the int of the SampleName, this can be removed to + /// do a string comp at some point. + /// @param i parameter index + /// @ingroup ParameterHandlerGetters + std::vector GetParSampleID(const int i) const { + return parlist.params.samples[i]; + }; + /// @brief ETA - just return a string of "spline", "norm" or "functional" + /// @param i parameter index + /// @ingroup ParameterHandlerGetters + std::string GetParamTypeString(const int i) const { + return SystType_ToString(GetParamType(i)); + } + + TypeParameterBase const *GetParam(const int i) const; + /// @brief Returns enum describing our param type + /// @param i parameter index + /// @ingroup ParameterHandlerGetters + SystType GetParamType(const int i) const; + + /// @brief Get interpolation type for a given parameter + /// @param i spline parameter index, not confuse with global index + SplineInterpolation GetParSplineInterpolation(const int i) const { + return SplineParams.at(i).SplineInterpolationType; + } + /// @brief Get the interpolation types for splines affecting a particular + /// SampleName + /// @ingroup ParameterHandlerGetters + const std::vector + GetSplineInterpolationFromSampleName(const std::string &SampleName); + /// @brief Get the name of the spline associated with the spline at index i + /// @param i spline parameter index, not to be confused with global index + /// @ingroup ParameterHandlerGetters + std::string GetParSplineName(const int i) const { + return SplineParams.at(i).spline_name; + } + + /// @brief DB Get spline parameters depending on given SampleName + /// @ingroup ParameterHandlerGetters + const std::vector + GetGlobalSystIndexFromSampleName(const std::string &SampleName, + const SystType Type); + /// @brief EM: value at which we cap spline knot weight + /// @param i spline parameter index, not confuse with global index + /// @ingroup ParameterHandlerGetters + double GetParSplineKnotUpperBound(const int i) const { + return SplineParams.at(i).SplineKnotUpBound; + } + /// @brief EM: value at which we cap spline knot weight + /// @param i spline parameter index, not confuse with global index + /// @ingroup ParameterHandlerGetters + double GetParSplineKnotLowerBound(const int i) const { + return SplineParams.at(i).SplineKnotLowBound; + } + + /// @brief DB Grab the number of parameters for the relevant SampleName + /// @param SampleName property of SampleHandler class based on which we select + /// whether to apply uncertainties or not + /// @param Type Type of syst, for example kNorm, kSpline etc + /// @ingroup ParameterHandlerGetters + int GetNumParamsFromSampleName(const std::string &SampleName, + const SystType Type); + /// @brief DB Grab the parameter names for the relevant SampleName + /// @param SampleName property of SampleHandler class based on which we select + /// whether to apply uncertainties or not + /// @param Type Type of syst, for example kNorm, kSpline etc + /// @ingroup ParameterHandlerGetters + const std::vector + GetParsNamesFromSampleName(const std::string &SampleName, + const SystType Type); + /// @brief DB Grab the parameter indices for the relevant SampleName + /// @param SampleName property of SampleHandler class based on which we select + /// whether to apply uncertainties or not + /// @param Type Type of syst, for example kNorm, kSpline etc + /// @ingroup ParameterHandlerGetters + const std::vector + GetParsIndexFromSampleName(const std::string &SampleName, + const SystType Type); + + /// @brief DB Get spline parameters depending on given SampleName + /// @ingroup ParameterHandlerGetters + const std::vector + GetSplineParsNamesFromSampleName(const std::string &SampleName); + /// @brief DB Get spline parameters depending on given SampleName + /// @ingroup ParameterHandlerGetters + const std::vector + GetSplineFileParsNamesFromSampleName(const std::string &SampleName); + + /// @brief DB Grab the Spline Modes for the relevant SampleName + /// @ingroup ParameterHandlerGetters + const std::vector> + GetSplineModeVecFromSampleName(const std::string &SampleName); + + /// @brief DB Get norm/func parameters depending on given SampleName + /// @ingroup ParameterHandlerGetters + const std::vector + GetNormParsFromSampleName(const std::string &SampleName) const; + /// @brief HH Get functional parameters for the relevant SampleName + /// @ingroup ParameterHandlerGetters + const std::vector + GetFunctionalParametersFromSampleName(const std::string &SampleName) const; + /// @brief KS: Grab the Spline parameters for the relevant SampleName + /// @ingroup ParameterHandlerGetters + const std::vector + GetSplineParsFromSampleName(const std::string &SampleName) const; + + /// @brief Checks if parameter belongs to a given group + /// @param i parameter index + /// @param Group name of group, like Xsec or Flux + /// @return bool telling whether param is part of group + /// @ingroup ParameterHandlerGetters + bool IsParFromGroup(const int i, const std::string &Group) const; + + /// @brief KS: Check how many parameters are associated with given group + /// @ingroup ParameterHandlerGetters + int GetNumParFromGroup(const std::string &Group) const; + /// @brief KS: Get names of all unique parameter groups + /// @ingroup ParameterHandlerGetters + std::vector GetUniqueParameterGroups(); + + /// @brief KS Function to set to prior parameters of a given group or values + /// from vector + /// @param Group name of group, like Xsec or Flux + /// @param Pars Values which will overwrite proposed step + /// @ingroup ParameterHandlerSetters + /// @note this mimic functionality of @ParameterHandlerBase::SetParameters + void SetGroupOnlyParameters(const std::string &Group, + const std::vector &Pars = {}); + /// @brief KS Function to set to prior parameters of a given groups or values + /// from vector + /// @param Groups vector of group names, like Xsec or Flux + /// @ingroup ParameterHandlerSetters + void SetGroupOnlyParameters(const std::vector &Groups); + + /// @brief TN Method to set parameters within a group to be fixed to their + /// prior values + /// @param Group name of the parameter group (Xsec, Flux, Osc, etc.) + void SetFixGroupOnlyParameters(const std::string &Group); + /// @brief TN Method to set parameters of certain groups to be fixed to their + /// prior values + /// @param Groups vector of group names (e.g. {"Xsec", "Flux"}) + void SetFixGroupOnlyParameters(const std::vector &Groups); + + /// @brief TN Method to set parameters within a group to be treated as free + /// @param Group name of the parameter group (Xsec, Flux, Osc, etc.) + void SetFreeGroupOnlyParameters(const std::string &Group); + /// @brief TN Method to set parameters of certain groups to be treated as free + /// @param Groups vector of group names (e.g. {"Xsec", "Flux"}) + void SetFreeGroupOnlyParameters(const std::vector &Groups); + + /// @brief TN Method to toggle fix/free parameters within a group + /// @param Group name of the parameter group (Xsec, Flux, Osc, etc.) + void ToggleFixGroupOnlyParameters(const std::string &Group); + /// @brief TN Method to toggle fix/free parameters within given groups + /// @param Group vector of group names (e.g. {"Xsec", "Flux"}) + void ToggleFixGroupOnlyParameters(const std::vector &Groups); + + /// @brief Dump Matrix to ROOT file, useful when we need to pass matrix info + /// to another fitting group + /// @param Name Name of TFile to which we save stuff + /// @warning This is mostly used for backward compatibility + void DumpMatrixToFile(const std::string &Name); + + /// @brief Get pointers to Osc params from Sample name + /// @ingroup ParameterHandlerGetters + std::vector + GetOscParsFromSampleName(const std::string &SampleName); + + void DetermineGlobalParameterIndices(); + +protected: + /// @brief Print information about the whole object once it is set + void Print(); + /// @brief Prints general information about the ParameterHandler object. + void PrintGlobablInfo(); + /// @brief Prints normalization parameters. + void PrintNormParams(); + /// @brief Prints spline parameters. + void PrintSplineParams(); + /// @brief Prints functional parameters. + void PrintFunctionalParams(); + /// @brief Prints oscillation parameters. + void PrintOscillationParams(); + /// @brief Prints groups of parameters. + void PrintParameterGroups(); + + /// @brief KS: Check if matrix is correctly initialised + void CheckCorrectInitialisation(); + + /// @brief Iterates over parameters and applies a filter and action function. + /// + /// This template function provides a way to iterate over parameters + /// associated with a specific Sample ID (SampleName). It applies a filter + /// function to determine which parameters to process and an action function + /// to define what to do with the selected parameters. + /// + /// @tparam FilterFunc The type of the filter function used to determine + /// which parameters to include. + /// @tparam ActionFunc The type of the action function applied to each + /// selected parameter. + /// @param SampleName The Sample ID used to filter parameters. + template + void IterateOverParams(const std::string &SampleName, + FilterFunc const &filter, ActionFunc const &action); + + /// @brief Initializes the systematic parameters from the configuration file. + /// This function loads parameters like normalizations and splines from the + /// provided YAML file. + /// @note This is used internally during the object's initialization process. + void InitParams(); + + /// @brief Parses the YAML configuration to set up cross-section parameters. + /// The YAML file defines the types of systematic errors, interpolation types, + /// and bounds for splines. + void InitParametersTypeFromConfig(); + + /// @brief Get Norm params + /// @param param Yaml node describing param + /// @param Index Global parameter index + NormParameter GetNormParameter(const YAML::Node ¶m); + + /// @brief Get Osc params + /// @param param Yaml node describing param + /// @param Index Global parameter index + OscillationParameter GetOscillationParameters(const YAML::Node ¶m); + + /// @brief Get Func params + /// @param param Yaml node describing param + /// @param Index Global parameter index + FunctionalParameter GetFunctionalParameters(const YAML::Node ¶m); + /// @brief Get Spline params + /// @param param Yaml node describing param + /// @param Index Global parameter index + SplineParameter GetSplineParameter(const YAML::Node ¶m); + + /// @brief Retrieve parameters that apply to a given sample name. + /// @tparam ParamT Type of parameter (e.g., FunctionalParameter, + /// NormParameter). + /// @param indexMap Map from local to global parameter indices. + /// @param params Vector of all parameters of the given type. + /// @param SampleName The name of the sample to filter applicable parameters + /// for. + /// @return Vector of parameters of type ParamT that apply to the specified + /// sample. + template + std::vector + GetTypeParamsFromSampleName(const std::map &indexMap, + const std::vector ¶ms, + const std::string &SampleName) const; + + std::vector GlobalParams; + + /// Vector containing info for normalisation systematics + std::vector SplineParams; + + /// Vector containing info for normalisation systematics + std::vector NormParams; + + /// Vector containing info for functional systematics + std::vector FuncParams; + + /// Vector containing info for functional systematics + std::vector OscParams; }; diff --git a/Parameters/ParameterStructs.h b/Parameters/ParameterStructs.h index a887294db..317d7aff5 100644 --- a/Parameters/ParameterStructs.h +++ b/Parameters/ParameterStructs.h @@ -1,57 +1,56 @@ #pragma once // C++ includes -#include #include +#include #include // MaCh3 includes +#include "Manager/Core.h" #include "Manager/MaCh3Exception.h" #include "Manager/MaCh3Logger.h" -#include "Manager/Core.h" _MaCh3_Safe_Include_Start_ //{ // ROOT include -#include "TSpline.h" -#include "TObjString.h" -#include "TFile.h" #include "TF1.h" -#include "TH2Poly.h" +#include "TFile.h" #include "TH1.h" -_MaCh3_Safe_Include_End_ //} - -/// @file ParameterStructs.h -/// @brief Definitions of generic parameter structs and utility templates for MaCh3. -/// @author Asher Kaboth -/// @author Clarence Wret -/// @author Patrick Dunne -/// @author Dan Barrow -/// @author Ed Atkin -/// @author Kamil Skwarczynski - -// ******************* -/// @brief Template to make vector out of an array of any length -template< typename T, size_t N > -std::vector MakeVector( const T (&data)[N] ) { -// ******************* - return std::vector(data, data+N); +#include "TH2Poly.h" +#include "TObjString.h" +#include "TSpline.h" + _MaCh3_Safe_Include_End_ //} + + /// @file ParameterStructs.h + /// @brief Definitions of generic parameter structs and utility templates + /// for MaCh3. + /// @author Asher Kaboth + /// @author Clarence Wret + /// @author Patrick Dunne + /// @author Dan Barrow + /// @author Ed Atkin + /// @author Kamil Skwarczynski + + // ******************* + /// @brief Template to make vector out of an array of any length + template + std::vector MakeVector(const T (&data)[N]) { + // ******************* + return std::vector(data, data + N); } // ******************* /// @brief Generic cleanup function -template -void CleanVector(std::vector& vec) { -// ******************* +template void CleanVector(std::vector &vec) { + // ******************* vec.clear(); vec.shrink_to_fit(); } // ******************* /// @brief Generic cleanup function -template -void CleanVector(std::vector>& vec) { -// ******************* - for (auto& innerVec : vec) { +template void CleanVector(std::vector> &vec) { + // ******************* + for (auto &innerVec : vec) { innerVec.clear(); innerVec.shrink_to_fit(); } @@ -62,10 +61,10 @@ void CleanVector(std::vector>& vec) { // ******************* /// @brief Generic cleanup function template -void CleanVector(std::vector>>& vec) { -// ******************* - for (auto& inner2DVec : vec) { - for (auto& innerVec : inner2DVec) { +void CleanVector(std::vector>> &vec) { + // ******************* + for (auto &inner2DVec : vec) { + for (auto &innerVec : inner2DVec) { innerVec.clear(); innerVec.shrink_to_fit(); } @@ -80,14 +79,17 @@ void CleanVector(std::vector>>& vec) { /// @brief Generic cleanup function /// @todo Use recursive to make it more scalable in future template -void CleanVector(std::vector>>>>>> &vec) { -// ******************* - for (auto& v6 : vec) { - for (auto& v5 : v6) { - for (auto& v4 : v5) { - for (auto& v3 : v4) { - for (auto& v2 : v3) { - for (auto& v1 : v2) { +void CleanVector( + std::vector>>>>>> + &vec) { + // ******************* + for (auto &v6 : vec) { + for (auto &v5 : v6) { + for (auto &v4 : v5) { + for (auto &v3 : v4) { + for (auto &v2 : v3) { + for (auto &v1 : v2) { v1.clear(); v1.shrink_to_fit(); } @@ -112,11 +114,11 @@ void CleanVector(std::vector -void CleanContainer(std::vector& container) { -// ******************* - for (T* ptr : container) { - if(ptr) delete ptr; +template void CleanContainer(std::vector &container) { + // ******************* + for (T *ptr : container) { + if (ptr) + delete ptr; ptr = nullptr; } container.clear(); @@ -126,9 +128,9 @@ void CleanContainer(std::vector& container) { // ******************* /// @brief Generic cleanup function template -void CleanContainer(std::vector>& container) { -// ******************* - for (auto& inner : container) { +void CleanContainer(std::vector> &container) { + // ******************* + for (auto &inner : container) { CleanContainer(inner); } container.clear(); @@ -138,12 +140,13 @@ void CleanContainer(std::vector>& container) { // ******************* /// @brief Generic cleanup function template -void CleanContainer(std::vector< std::vector< std::vector > >& container) { -// ******************* - for (auto& container2D : container) { - for (auto& container1D : container2D) { - for (T* ptr : container1D) { - if(ptr) delete ptr; +void CleanContainer(std::vector>> &container) { + // ******************* + for (auto &container2D : container) { + for (auto &container1D : container2D) { + for (T *ptr : container1D) { + if (ptr) + delete ptr; } container1D.clear(); container1D.shrink_to_fit(); @@ -155,12 +158,27 @@ void CleanContainer(std::vector< std::vector< std::vector > >& container) { container.shrink_to_fit(); } +/// Make an enum of systematic type recognised by covariance class +/// @todo KS: Consider using enum class, it is generally recommended as safer. +/// It will require many static_cast +enum SystType { + kNorm, //!< For normalisation parameters + kSpline, //!< For splined parameters (1D) + kFunc, //!< For functional parameters + kOsc, //!< For oscillation parameters + kSystTypes //!< This only enumerates +}; + // ******************* /// @brief Base class storing info for parameters types, helping unify codebase struct TypeParameterBase { -// ******************* + // ******************* /// Name of parameters - std::string name; + std::string fancy_name; + + std::string group; + + SystType syst_type; /// Parameter number of this normalisation in current systematic model int index = M3::_BAD_INT_; @@ -170,6 +188,9 @@ struct TypeParameterBase { /// @brief ETA - Normalisations for cross-section parameters /// Carrier for whether you want to apply a systematic to an event or not struct NormParameter : public TypeParameterBase { + + NormParameter() { syst_type = kNorm; } + /// Mode which parameter applies to std::vector modes; /// Horn currents which parameter applies to @@ -184,22 +205,25 @@ struct NormParameter : public TypeParameterBase { bool hasKinBounds; /// Generic vector contain enum relating to a kinematic variable /// and lower and upper bounds. This can then be passed to IsEventSelected - std::vector< std::vector< std::vector > > Selection; + std::vector>> Selection; /// Generic vector containing the string of kinematic type /// This then needs to be converted to a kinematic type enum /// within a SampleHandler daughter class /// The bounds for each kinematic variable are given in Selection - std::vector< std::string > KinematicVarStr; + std::vector KinematicVarStr; }; // HH - a shorthand type for funcpar functions -using FuncParFuncType = std::function; +using FuncParFuncType = std::function; // ******************* /// @brief HH - Functional parameters /// Carrier for whether you want to apply a systematic to an event or not struct FunctionalParameter : public TypeParameterBase { -// ******************* + // ******************* + + FunctionalParameter() { syst_type = kFunc; } + /// Mode which parameter applies to std::vector modes; /// Horn currents which parameter applies to @@ -214,37 +238,40 @@ struct FunctionalParameter : public TypeParameterBase { bool hasKinBounds; /// Generic vector contain enum relating to a kinematic variable /// and lower and upper bounds. This can then be passed to IsEventSelected - std::vector< std::vector< std::vector > > Selection; + std::vector>> Selection; /// Generic vector containing the string of kinematic type /// This then needs to be converted to a kinematic type enum /// within a SampleHandler daughter class /// The bounds for each kinematic variable are given in Selection - std::vector< std::string > KinematicVarStr; + std::vector KinematicVarStr; /// Parameter value pointer - const double* valuePtr; + const double *valuePtr; /// Function pointer - FuncParFuncType* funcPtr; + FuncParFuncType *funcPtr; }; /// Make an enum of the spline interpolation type enum RespFuncType { - kTSpline3_red, //!< Uses TSpline3_red for interpolation - kTF1_red, //!< Uses TF1_red for interpolation - kRespFuncTypes //!< This only enumerates + kTSpline3_red, //!< Uses TSpline3_red for interpolation + kTF1_red, //!< Uses TF1_red for interpolation + kRespFuncTypes //!< This only enumerates }; /// Make an enum of the spline interpolation type enum SplineInterpolation { - kTSpline3, //!< Default TSpline3 interpolation - kLinear, //!< Linear interpolation between knots - kMonotonic, //!< EM: DOES NOT make the entire spline monotonic, only the segments - kAkima, //!< EM: Akima spline iis allowed to be discontinuous in 2nd derivative and coefficients in any segment - kKochanekBartels, //!< KS: Kochanek-Bartels spline: allows local control of tension, continuity, and bias - kLinearFunc, //!< Liner interpolation using TF1 not spline - kSplineInterpolations //!< This only enumerates + kTSpline3, //!< Default TSpline3 interpolation + kLinear, //!< Linear interpolation between knots + kMonotonic, //!< EM: DOES NOT make the entire spline monotonic, only the + //!< segments + kAkima, //!< EM: Akima spline iis allowed to be discontinuous in 2nd + //!< derivative and coefficients in any segment + kKochanekBartels, //!< KS: Kochanek-Bartels spline: allows local control of + //!< tension, continuity, and bias + kLinearFunc, //!< Liner interpolation using TF1 not spline + kSplineInterpolations //!< This only enumerates }; // ************************************************** @@ -253,22 +280,23 @@ enum SplineInterpolation { inline std::string GetTF1(const SplineInterpolation i) { // ************************************************** std::string Func = ""; - switch(i) { - case SplineInterpolation::kLinearFunc: - Func = "([1]+[0]*x)"; - break; - case SplineInterpolation::kTSpline3: - case SplineInterpolation::kLinear: - case SplineInterpolation::kMonotonic: - case SplineInterpolation::kAkima: - case SplineInterpolation::kKochanekBartels: - case SplineInterpolation::kSplineInterpolations: - MACH3LOG_ERROR("Interpolation type {} not supported for TF1!", static_cast(i)); - throw MaCh3Exception(__FILE__, __LINE__); - default: - MACH3LOG_ERROR("UNKNOWN SPLINE INTERPOLATION SPECIFIED!"); - MACH3LOG_ERROR("You gave {}", static_cast(i)); - throw MaCh3Exception(__FILE__ , __LINE__ ); + switch (i) { + case SplineInterpolation::kLinearFunc: + Func = "([1]+[0]*x)"; + break; + case SplineInterpolation::kTSpline3: + case SplineInterpolation::kLinear: + case SplineInterpolation::kMonotonic: + case SplineInterpolation::kAkima: + case SplineInterpolation::kKochanekBartels: + case SplineInterpolation::kSplineInterpolations: + MACH3LOG_ERROR("Interpolation type {} not supported for TF1!", + static_cast(i)); + throw MaCh3Exception(__FILE__, __LINE__); + default: + MACH3LOG_ERROR("UNKNOWN SPLINE INTERPOLATION SPECIFIED!"); + MACH3LOG_ERROR("You gave {}", static_cast(i)); + throw MaCh3Exception(__FILE__, __LINE__); } return Func; } @@ -276,27 +304,28 @@ inline std::string GetTF1(const SplineInterpolation i) { // ************************************************** /// @brief Convert a RespFuncType type to a SplineInterpolation /// @param i Interpolation type -inline RespFuncType SplineInterpolation_ToRespFuncType(const SplineInterpolation i) { -// ************************************************** +inline RespFuncType +SplineInterpolation_ToRespFuncType(const SplineInterpolation i) { + // ************************************************** RespFuncType Type = kRespFuncTypes; - switch(i) { - case SplineInterpolation::kTSpline3: - case SplineInterpolation::kLinear: - case SplineInterpolation::kMonotonic: - case SplineInterpolation::kAkima: - case SplineInterpolation::kKochanekBartels: - Type = RespFuncType::kTSpline3_red; - break; - case SplineInterpolation::kLinearFunc: - Type = RespFuncType::kTF1_red; - break; - case SplineInterpolation::kSplineInterpolations: - MACH3LOG_ERROR("kSplineInterpolations is not a valid interpolation type!"); - throw MaCh3Exception(__FILE__, __LINE__); - default: - MACH3LOG_ERROR("UNKNOWN SPLINE INTERPOLATION SPECIFIED!"); - MACH3LOG_ERROR("You gave {}", static_cast(i)); - throw MaCh3Exception(__FILE__, __LINE__); + switch (i) { + case SplineInterpolation::kTSpline3: + case SplineInterpolation::kLinear: + case SplineInterpolation::kMonotonic: + case SplineInterpolation::kAkima: + case SplineInterpolation::kKochanekBartels: + Type = RespFuncType::kTSpline3_red; + break; + case SplineInterpolation::kLinearFunc: + Type = RespFuncType::kTF1_red; + break; + case SplineInterpolation::kSplineInterpolations: + MACH3LOG_ERROR("kSplineInterpolations is not a valid interpolation type!"); + throw MaCh3Exception(__FILE__, __LINE__); + default: + MACH3LOG_ERROR("UNKNOWN SPLINE INTERPOLATION SPECIFIED!"); + MACH3LOG_ERROR("You gave {}", static_cast(i)); + throw MaCh3Exception(__FILE__, __LINE__); } return Type; } @@ -304,99 +333,109 @@ inline RespFuncType SplineInterpolation_ToRespFuncType(const SplineInterpolation // ************************************************** /// @brief Convert a LLH type to a string inline std::string SplineInterpolation_ToString(const SplineInterpolation i) { -// ************************************************** + // ************************************************** std::string name = ""; - switch(i) { - // TSpline3 (third order spline in ROOT) - case SplineInterpolation::kTSpline3: - name = "TSpline3"; - break; - case SplineInterpolation::kLinear: - name = "Linear"; - break; - case SplineInterpolation::kMonotonic: - name = "Monotonic"; - break; - // (Experimental) Akima_Spline (crd order spline which is allowed to be discontinuous in 2nd deriv) - case SplineInterpolation::kAkima: - name = "Akima"; - break; - case SplineInterpolation::kKochanekBartels: - name = "KochanekBartels"; - break; - case SplineInterpolation::kLinearFunc: - name = "LinearFunc"; - break; - case SplineInterpolation::kSplineInterpolations: - MACH3LOG_ERROR("kSplineInterpolations is not a valid interpolation type!"); - throw MaCh3Exception(__FILE__, __LINE__); - default: - MACH3LOG_ERROR("UNKNOWN SPLINE INTERPOLATION SPECIFIED!"); - MACH3LOG_ERROR("You gave {}", static_cast(i)); - throw MaCh3Exception(__FILE__ , __LINE__ ); + switch (i) { + // TSpline3 (third order spline in ROOT) + case SplineInterpolation::kTSpline3: + name = "TSpline3"; + break; + case SplineInterpolation::kLinear: + name = "Linear"; + break; + case SplineInterpolation::kMonotonic: + name = "Monotonic"; + break; + // (Experimental) Akima_Spline (crd order spline which is allowed to be + // discontinuous in 2nd deriv) + case SplineInterpolation::kAkima: + name = "Akima"; + break; + case SplineInterpolation::kKochanekBartels: + name = "KochanekBartels"; + break; + case SplineInterpolation::kLinearFunc: + name = "LinearFunc"; + break; + case SplineInterpolation::kSplineInterpolations: + MACH3LOG_ERROR("kSplineInterpolations is not a valid interpolation type!"); + throw MaCh3Exception(__FILE__, __LINE__); + default: + MACH3LOG_ERROR("UNKNOWN SPLINE INTERPOLATION SPECIFIED!"); + MACH3LOG_ERROR("You gave {}", static_cast(i)); + throw MaCh3Exception(__FILE__, __LINE__); } return name; } -/// Make an enum of systematic type recognised by covariance class -/// @todo KS: Consider using enum class, it is generally recommended as safer. It will require many static_cast -enum SystType { - kNorm, //!< For normalisation parameters - kSpline, //!< For splined parameters (1D) - kFunc, //!< For functional parameters - kOsc, //!< For oscillation parameters - kSystTypes //!< This only enumerates -}; - // ******************* /// @brief KS: Struct holding info about Spline Systematics struct SplineParameter : public TypeParameterBase { -// ******************* + // ******************* + + SplineParameter() { syst_type = kSpline; }; + /// Spline interpolation vector - SplineInterpolation _SplineInterpolationType; + SplineInterpolation SplineInterpolationType; /// Modes to which spline applies (valid only for binned splines) - std::vector _fSplineModes; + std::vector SplineModes; + + std::string spline_name; /// EM: Cap spline knot lower value - double _SplineKnotLowBound; + double SplineKnotLowBound; /// EM: Cap spline knot higher value - double _SplineKnotUpBound; + double SplineKnotUpBound; }; // ************************************************** /// @brief Convert a Syst type type to a string inline std::string SystType_ToString(const SystType i) { -// ************************************************** + // ************************************************** std::string name = ""; - switch(i) { - case SystType::kNorm: - name = "Norm"; - break; - case SystType::kSpline: - name = "Spline"; - break; - case SystType::kFunc: - name = "Functional"; - break; - case SystType::kOsc: - name = "Oscillation"; - break; - case SystType::kSystTypes: - MACH3LOG_ERROR("kSystTypes is not a valid SystType!"); - throw MaCh3Exception(__FILE__, __LINE__); - default: - MACH3LOG_ERROR("UNKNOWN SYST TYPE SPECIFIED!"); - MACH3LOG_ERROR("You gave {}", static_cast(i)); - throw MaCh3Exception(__FILE__ , __LINE__ ); + switch (i) { + case SystType::kNorm: + name = "Norm"; + break; + case SystType::kSpline: + name = "Spline"; + break; + case SystType::kFunc: + name = "Functional"; + break; + case SystType::kOsc: + name = "Oscillation"; + break; + case SystType::kSystTypes: + MACH3LOG_ERROR("kSystTypes is not a valid SystType!"); + throw MaCh3Exception(__FILE__, __LINE__); + default: + MACH3LOG_ERROR("UNKNOWN SYST TYPE SPECIFIED!"); + MACH3LOG_ERROR("You gave {}", static_cast(i)); + throw MaCh3Exception(__FILE__, __LINE__); } return name; } +inline SystType String_ToSystType(std::string const &typestr) { + // LP this is hideous but for a few entries, its fine. + for (SystType st = kNorm; st < kSystTypes; st = SystType(st + 1)) { + if (SystType_ToString(st) == typestr) { + return st; + } + } + MACH3LOG_ERROR("Failed to parse SystType from {}", typestr); + throw MaCh3Exception(__FILE__, __LINE__); +} + // ******************* /// @brief KS: Struct holding info about oscillation Systematics /// @note right now it is empty struct OscillationParameter : public TypeParameterBase { -// ******************* + // ******************* + OscillationParameter() { syst_type = kOsc; }; + /// Parameter value pointer + const double *valuePtr; }; diff --git a/Samples/SampleHandlerFD.cpp b/Samples/SampleHandlerFD.cpp index ef6f334ce..aaab2bc84 100644 --- a/Samples/SampleHandlerFD.cpp +++ b/Samples/SampleHandlerFD.cpp @@ -45,7 +45,7 @@ SampleHandlerFD::SampleHandlerFD(std::string ConfigFileName, ParameterHandlerGen SampleHandlerFD::~SampleHandlerFD() { MACH3LOG_DEBUG("I'm deleting SampleHandlerFD"); - + if (SampleHandlerFD_array != nullptr) delete[] SampleHandlerFD_array; if (SampleHandlerFD_array_w2 != nullptr) delete[] SampleHandlerFD_array_w2; //ETA - there is a chance that you haven't added any data... @@ -309,9 +309,9 @@ bool SampleHandlerFD::IsSubEventSelected(const std::vector &SubEve // Reweight function void SampleHandlerFD::Reweight() { //************************************************ - //KS: Reset the histograms before reweight + //KS: Reset the histograms before reweight ResetHistograms(); - + //You only need to do these things if Oscillator has been initialised //if not then you're not considering oscillations if (Oscillator) Oscillator->Evaluate(); @@ -341,7 +341,7 @@ void SampleHandlerFD::FillArray() { //************************************************ //DB Reset which cuts to apply Selection = StoredSelection; - + PrepFunctionalParameters(); for (unsigned int iEvent = 0; iEvent < GetNEvents(); iEvent++) { @@ -377,7 +377,7 @@ void SampleHandlerFD::FillArray() { } #ifdef MULTITHREAD -// ************************************************ +// ************************************************ /// Multithreaded version of fillArray @see fillArray() void SampleHandlerFD::FillArray_MP() { // ************************************************ @@ -476,7 +476,7 @@ void SampleHandlerFD::FillArray_MP() { // ************************************************** // Helper function to reset the data and MC histograms void SampleHandlerFD::ResetHistograms() { -// ************************************************** +// ************************************************** //DB Reset values stored in PDF array to 0. // Don't openMP this; no significant gain #ifdef MULTITHREAD @@ -509,7 +509,7 @@ void SampleHandlerFD::RegisterIndividualFunctionalParameter(const std::string& f void SampleHandlerFD::SetupFunctionalParameters() { funcParsVec = ParHandler->GetFunctionalParametersFromSampleName(SampleHandlerName); - // RegisterFunctionalParameters is implemented in experiment-specific code, + // RegisterFunctionalParameters is implemented in experiment-specific code, // which calls RegisterIndividualFuncPar to populate funcParsNamesMap, funcParsNamesVec, and funcParsFuncMap RegisterFunctionalParameters(); funcParsMap.resize(funcParsNamesMap.size()); @@ -518,16 +518,16 @@ void SampleHandlerFD::SetupFunctionalParameters() { // For every functional parameter in XsecCov that matches the name in funcParsNames, add it to the map for (FunctionalParameter & fp : funcParsVec) { for (std::string name : funcParsNamesVec) { - if (fp.name == name) { - MACH3LOG_INFO("Adding functional parameter: {} to funcParsMap with key: {}", fp.name, funcParsNamesMap[fp.name]); - fp.funcPtr = &funcParsFuncMap[funcParsNamesMap[fp.name]]; - funcParsMap[static_cast(funcParsNamesMap[fp.name])] = &fp; + if (fp.fancy_name == name) { + MACH3LOG_INFO("Adding functional parameter: {} to funcParsMap with key: {}", fp.fancy_name, funcParsNamesMap[fp.fancy_name]); + fp.funcPtr = &funcParsFuncMap[funcParsNamesMap[fp.fancy_name]]; + funcParsMap[static_cast(funcParsNamesMap[fp.fancy_name])] = &fp; continue; } } // If we don't find a match, we need to throw an error - if (funcParsMap[static_cast(funcParsNamesMap[fp.name])] == nullptr) { - MACH3LOG_ERROR("Functional parameter {} not found, did you define it in RegisterFunctionalParameters()?", fp.name); + if (funcParsMap[static_cast(funcParsNamesMap[fp.fancy_name])] == nullptr) { + MACH3LOG_ERROR("Functional parameter {} not found, did you define it in RegisterFunctionalParameters()?", fp.fancy_name); throw MaCh3Exception(__FILE__, __LINE__); } } @@ -575,7 +575,7 @@ void SampleHandlerFD::SetupFunctionalParameters() { MACH3LOG_TRACE("Event {}, missed Kinematic var check for dial {}", iEvent, (*it).name); continue; } - auto funcparenum = funcParsNamesMap[(*it).name]; + auto funcparenum = funcParsNamesMap[(*it).fancy_name]; funcParsGrid.at(iEvent).push_back(funcparenum); } } @@ -1048,11 +1048,11 @@ void SampleHandlerFD::AddData(const int Sample, TH1D* Data) { SampleDetails[Sample].dathist = static_cast(Data->Clone()); if (GetNDim(Sample) != 1) { - MACH3LOG_ERROR("Trying to set a 1D 'data' histogram in a 2D sample - Quitting"); + MACH3LOG_ERROR("Trying to set a 1D 'data' histogram in a 2D sample - Quitting"); MACH3LOG_ERROR("The number of dimensions for this sample is {}", GetNDim(Sample)); throw MaCh3Exception(__FILE__ , __LINE__ ); } - + if(SampleHandlerFD_data == nullptr) { MACH3LOG_ERROR("SampleHandlerFD_data haven't been initialised yet"); throw MaCh3Exception(__FILE__, __LINE__); @@ -1076,9 +1076,9 @@ void SampleHandlerFD::AddData(const int Sample, TH2D* Data) { SampleDetails[Sample].dathist = nullptr; if (GetNDim(Sample) != 2) { - MACH3LOG_ERROR("Trying to set a 2D 'data' histogram in a 1D sample - Quitting"); + MACH3LOG_ERROR("Trying to set a 2D 'data' histogram in a 1D sample - Quitting"); throw MaCh3Exception(__FILE__ , __LINE__ );} - + if(SampleHandlerFD_data == nullptr) { MACH3LOG_ERROR("SampleHandlerFD_data haven't been initialised yet"); throw MaCh3Exception(__FILE__, __LINE__); @@ -1299,6 +1299,7 @@ double SampleHandlerFD::GetSampleLikelihood(const int isample) const { // ************************************************ double SampleHandlerFD::GetLikelihood() const { // ************************************************ + double negLogL = 0.; #ifdef MULTITHREAD #pragma omp parallel for reduction(+:negLogL) @@ -1531,13 +1532,13 @@ TH2* SampleHandlerFD::Get2DVarHist(const int iSample, const std::string& Project void SampleHandlerFD::Fill2DSubEventHist(const int iSample, TH2D* _h2DVar, const std::string& ProjectionVar_StrX, const std::string& ProjectionVar_StrY, const std::vector< KinematicCut >& SubEventSelectionVec, int WeightStyle) { bool IsSubEventVarX = IsSubEventVarString(ProjectionVar_StrX); - bool IsSubEventVarY = IsSubEventVarString(ProjectionVar_StrY); + bool IsSubEventVarY = IsSubEventVarString(ProjectionVar_StrY); int ProjectionVar_IntX, ProjectionVar_IntY; if (IsSubEventVarX) ProjectionVar_IntX = ReturnKinematicVectorFromString(ProjectionVar_StrX); else ProjectionVar_IntX = ReturnKinematicParameterFromString(ProjectionVar_StrX); if (IsSubEventVarY) ProjectionVar_IntY = ReturnKinematicVectorFromString(ProjectionVar_StrY); - else ProjectionVar_IntY = ReturnKinematicParameterFromString(ProjectionVar_StrY); + else ProjectionVar_IntY = ReturnKinematicParameterFromString(ProjectionVar_StrY); //JM Loop over all events for (unsigned int iEvent = 0; iEvent < GetNEvents(); iEvent++) { @@ -1578,7 +1579,7 @@ void SampleHandlerFD::Fill2DSubEventHist(const int iSample, TH2D* _h2DVar, const if (IsSubEventVarY) VarY = VecY[iSubEvent]; _h2DVar->Fill(VarX,VarY,Weight); } - } + } } } } From 6af82bd0d9c4f8e76ed5cdf976fcec17092581bb Mon Sep 17 00:00:00 2001 From: Luke Pickering Date: Thu, 18 Dec 2025 16:51:12 +0000 Subject: [PATCH 08/14] update some parameters design rationale --- Parameters/README.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Parameters/README.md b/Parameters/README.md index f6fa5c330..a30166e14 100644 --- a/Parameters/README.md +++ b/Parameters/README.md @@ -3,12 +3,18 @@ ## ParameterHandlerBase Design Decisions Elementary objects needed for sampling: -- Parameter list - - Read from YAML. Knows everything about each systematic parameter: name, error, possibly-correlated prior, restrictions on parameter applicability +- ParameterList + - Read from YAML. Knows everything about each systematic parameter: name, error, possibly-correlated prior, restrictions on parameter applicability etc... + - Optionally performs basis translation and dimensionality truncation via block diagonal PCA and retains the transformations between the 'Systematic Parameter Basis' and the 'Proposer Parameter Basis' (possibly PCA'd). - StepProposer - - Knows nothing about the meaning of parameters - - Generates parameter values that are steps away from some current point according to a proposal matrix. + - Knows nothing about the meaning of parameters. + - Generates parameter values that are steps away from some current point according to a symmetric proposal function. In this case specified fully by a multivariate gaussian. - Keeps track of parameter step scales and possibly updates the proposal matrix as steps are accepted. - Possibly steps in a rotated parameter space. - Possibly has 'special proposals': flips and circular + - Does not need to know how to translate from its parameter basis back to the systematic parameter basis, that is the job of ParameterList + + +Open Questions: +- Should the likelihood be calculated in the systematic (full) parameter space or the proposal (possibly truncated) parameter space. NPar might differ significantly? From 3e06c729ad31b963d902b9b54fe527dda70dc3d7 Mon Sep 17 00:00:00 2001 From: Luke Pickering Date: Thu, 18 Dec 2025 16:54:29 +0000 Subject: [PATCH 09/14] updates fitter code to work with new parameterhandler --- Fitters/FitterBase.cpp | 1221 +++++++++++++++------------ Fitters/LikelihoodFit.cpp | 152 ++-- Fitters/LikelihoodFit.h | 2 + Fitters/MCMCBase.cpp | 303 ++++--- Fitters/MinuitFit.cpp | 197 ++--- Fitters/PSO.cpp | 484 ++++++----- Fitters/PredictiveThrower.cpp | 668 ++++++++------- Parameters/CMakeLists.txt | 2 +- Parameters/ParameterHandlerBase.cpp | 91 +- Parameters/ParameterHandlerBase.h | 23 +- 10 files changed, 1675 insertions(+), 1468 deletions(-) diff --git a/Fitters/FitterBase.cpp b/Fitters/FitterBase.cpp index dc5338df6..be56f5a51 100644 --- a/Fitters/FitterBase.cpp +++ b/Fitters/FitterBase.cpp @@ -2,22 +2,24 @@ #include "Samples/SampleHandlerFD.h" _MaCh3_Safe_Include_Start_ //{ +#include "TGraphAsymmErrors.h" #include "TRandom.h" #include "TStopwatch.h" #include "TTree.h" -#include "TGraphAsymmErrors.h" -_MaCh3_Safe_Include_End_ //} + _MaCh3_Safe_Include_End_ //} #pragma GCC diagnostic ignored "-Wuseless-cast" -// ************************* -// Initialise the manager and make it an object of FitterBase class -// Now we can dump manager settings to the output file -FitterBase::FitterBase(manager * const man) : fitMan(man) { -// ************************* + // ************************* + // Initialise the manager and make it an object of FitterBase class + // Now we can dump manager settings to the output file + FitterBase::FitterBase(manager *const man) + : fitMan(man) { + // ************************* AlgorithmName = ""; - //Get mach3 modes from manager - random = std::make_unique(Get(fitMan->raw()["General"]["Seed"], __FILE__, __LINE__)); + // Get mach3 modes from manager + random = std::make_unique( + Get(fitMan->raw()["General"]["Seed"], __FILE__, __LINE__)); // Counter of the accepted # of steps accCount = 0; @@ -26,50 +28,59 @@ FitterBase::FitterBase(manager * const man) : fitMan(man) { clock = std::make_unique(); stepClock = std::make_unique(); - #ifdef DEBUG +#ifdef DEBUG // Fit summary and debug info - debug = GetFromManager(fitMan->raw()["General"]["Debug"], false, __FILE__ , __LINE__); - #endif + debug = GetFromManager(fitMan->raw()["General"]["Debug"], false, + __FILE__, __LINE__); +#endif - auto outfile = Get(fitMan->raw()["General"]["OutputFile"], __FILE__ , __LINE__); + auto outfile = Get(fitMan->raw()["General"]["OutputFile"], + __FILE__, __LINE__); // Save output every auto_save steps - //you don't want this too often https://root.cern/root/html606/TTree_8cxx_source.html#l01229 - auto_save = Get(fitMan->raw()["General"]["MCMC"]["AutoSave"], __FILE__ , __LINE__); + // you don't want this too often + // https://root.cern/root/html606/TTree_8cxx_source.html#l01229 + auto_save = Get(fitMan->raw()["General"]["MCMC"]["AutoSave"], __FILE__, + __LINE__); // Set the output file outputFile = M3::Open(outfile, "RECREATE", __FILE__, __LINE__); outputFile->cd(); // Set output tree outTree = new TTree("posteriors", "Posterior_Distributions"); - // Auto-save every 200MB, the bigger the better https://root.cern/root/html606/TTree_8cxx_source.html#l01229 + // Auto-save every 200MB, the bigger the better + // https://root.cern/root/html606/TTree_8cxx_source.html#l01229 outTree->SetAutoSave(-200E6); FileSaved = false; SettingsSaved = false; OutputPrepared = false; - //Create TDirectory + // Create TDirectory CovFolder = outputFile->mkdir("CovarianceFolder"); outputFile->cd(); SampleFolder = outputFile->mkdir("SampleFolder"); outputFile->cd(); - #ifdef DEBUG +#ifdef DEBUG // Prepare the output log file - if (debug) debugFile.open((outfile+".log").c_str()); - #endif + if (debug) + debugFile.open((outfile + ".log").c_str()); +#endif TotalNSamples = 0; - fTestLikelihood = GetFromManager(fitMan->raw()["General"]["Fitter"]["FitTestLikelihood"], false, __FILE__ , __LINE__); + fTestLikelihood = GetFromManager( + fitMan->raw()["General"]["Fitter"]["FitTestLikelihood"], false, __FILE__, + __LINE__); } // ************************* // Destructor: close the logger and output file FitterBase::~FitterBase() { -// ************************* + // ************************* SaveOutput(); - if(outputFile != nullptr) delete outputFile; + if (outputFile != nullptr) + delete outputFile; outputFile = nullptr; MACH3LOG_DEBUG("Closing MaCh3 Fitter Engine"); } @@ -77,29 +88,31 @@ FitterBase::~FitterBase() { // ******************* // Prepare the output tree void FitterBase::SaveSettings() { -// ******************* - if(SettingsSaved) return; + // ******************* + if (SettingsSaved) + return; outputFile->cd(); - TDirectory* MaCh3Version = outputFile->mkdir("MaCh3Engine"); + TDirectory *MaCh3Version = outputFile->mkdir("MaCh3Engine"); MaCh3Version->cd(); if (std::getenv("MaCh3_ROOT") == NULL) { MACH3LOG_ERROR("Need MaCh3_ROOT environment variable"); MACH3LOG_ERROR("Please remember about source bin/setup.MaCh3.sh"); - throw MaCh3Exception(__FILE__ , __LINE__ ); + throw MaCh3Exception(__FILE__, __LINE__); } if (std::getenv("MACH3") == NULL) { MACH3LOG_ERROR("Need MACH3 environment variable"); - throw MaCh3Exception(__FILE__ , __LINE__ ); + throw MaCh3Exception(__FILE__, __LINE__); } std::string header_path = std::string(std::getenv("MACH3")); header_path += "/version.h"; - FILE* file = fopen(header_path.c_str(), "r"); - //KS: It is better to use experiment specific header file. If given experiment didn't provide it we gonna use one given by Core MaCh3. + FILE *file = fopen(header_path.c_str(), "r"); + // KS: It is better to use experiment specific header file. If given + // experiment didn't provide it we gonna use one given by Core MaCh3. if (!file) { header_path = std::string(std::getenv("MaCh3_ROOT")); header_path += "/version.h"; @@ -112,10 +125,10 @@ void FitterBase::SaveSettings() { versionHeader.ReadFile(header_path.c_str()); versionHeader.Write(); - if(GetName() == ""){ + if (GetName() == "") { MACH3LOG_ERROR("Name of currently used algorithm is {}", GetName()); MACH3LOG_ERROR("Have you forgotten to modify AlgorithmName?"); - throw MaCh3Exception(__FILE__ , __LINE__ ); + throw MaCh3Exception(__FILE__, __LINE__); } TNamed Engine(GetName(), GetName()); Engine.Write(GetName().c_str()); @@ -127,21 +140,29 @@ void FitterBase::SaveSettings() { fitMan->SaveSettings(outputFile); - MACH3LOG_WARN("\033[0;31mCurrent Total RAM usage is {:.2f} GB\033[0m", MaCh3Utils::getValue("VmRSS") / 1048576.0); - MACH3LOG_WARN("\033[0;31mOut of Total available RAM {:.2f} GB\033[0m", MaCh3Utils::getValue("MemTotal") / 1048576.0); + MACH3LOG_WARN("\033[0;31mCurrent Total RAM usage is {:.2f} GB\033[0m", + MaCh3Utils::getValue("VmRSS") / 1048576.0); + MACH3LOG_WARN("\033[0;31mOut of Total available RAM {:.2f} GB\033[0m", + MaCh3Utils::getValue("MemTotal") / 1048576.0); MACH3LOG_INFO("#####Current Setup#####"); MACH3LOG_INFO("Number of covariances: {}", systematics.size()); - for(unsigned int i = 0; i < systematics.size(); ++i) - MACH3LOG_INFO("{}: Cov name: {}, it has {} params", i, systematics[i]->GetName(), systematics[i]->GetNumParams()); + for (unsigned int i = 0; i < systematics.size(); ++i) + MACH3LOG_INFO("{}: Cov name: {}, it has {} params", i, + systematics[i]->GetName(), + systematics[i]->GetNumSystematicParams()); MACH3LOG_INFO("Number of SampleHandlers: {}", samples.size()); - for(unsigned int i = 0; i < samples.size(); ++i) { - MACH3LOG_INFO("{}: SampleHandler name: {}, it has {} samples",i , samples[i]->GetName(), samples[i]->GetNsamples()); - for(int iSam = 0; iSam < samples[i]->GetNsamples(); ++iSam) { - MACH3LOG_INFO(" {}: Sample name: {}, with {} osc channels",iSam , samples[i]->GetSampleTitle(iSam), samples[i]->GetNOscChannels(iSam)); + for (unsigned int i = 0; i < samples.size(); ++i) { + MACH3LOG_INFO("{}: SampleHandler name: {}, it has {} samples", i, + samples[i]->GetName(), samples[i]->GetNsamples()); + for (int iSam = 0; iSam < samples[i]->GetNsamples(); ++iSam) { + MACH3LOG_INFO(" {}: Sample name: {}, with {} osc channels", iSam, + samples[i]->GetSampleTitle(iSam), + samples[i]->GetNOscChannels(iSam)); } } - //TN: Have to close the folder in order to write it to disk before SaveOutput is called in the destructor + // TN: Have to close the folder in order to write it to disk before SaveOutput + // is called in the destructor CovFolder->Close(); SampleFolder->Close(); @@ -151,28 +172,33 @@ void FitterBase::SaveSettings() { // ******************* // Prepare the output tree void FitterBase::PrepareOutput() { -// ******************* - if(OutputPrepared) return; - //MS: Check if we are fitting the test likelihood, rather than T2K likelihood, and only setup T2K output if not - if(!fTestLikelihood) - { + // ******************* + if (OutputPrepared) + return; + // MS: Check if we are fitting the test likelihood, rather than T2K + // likelihood, and only setup T2K output if not + if (!fTestLikelihood) { // Check that we have added samples if (!samples.size()) { MACH3LOG_CRITICAL("No samples Found! Is this really what you wanted?"); - //throw MaCh3Exception(__FILE__ , __LINE__ ); + // throw MaCh3Exception(__FILE__ , __LINE__ ); } - // Do we want to save proposal? This will break plotting scripts and is heave for disk space and step time. Only use when debugging - bool SaveProposal = GetFromManager(fitMan->raw()["General"]["SaveProposal"], false, __FILE__ , __LINE__); + // Do we want to save proposal? This will break plotting scripts and is + // heave for disk space and step time. Only use when debugging + bool SaveProposal = GetFromManager( + fitMan->raw()["General"]["SaveProposal"], false, __FILE__, __LINE__); - if(SaveProposal) MACH3LOG_INFO("Will save in the chain proposal parameters and LogL"); + if (SaveProposal) + MACH3LOG_INFO("Will save in the chain proposal parameters and LogL"); // Prepare the output trees for (ParameterHandlerBase *cov : systematics) { cov->SetBranches(*outTree, SaveProposal); } outTree->Branch("LogL", &logLCurr, "LogL/D"); - if(SaveProposal) outTree->Branch("LogLProp", &logLProp, "LogLProp/D"); + if (SaveProposal) + outTree->Branch("LogLProp", &logLProp, "LogLProp/D"); outTree->Branch("accProb", &accProb, "accProb/D"); outTree->Branch("step", &step, "step/i"); outTree->Branch("stepTime", &stepTime, "stepTime/D"); @@ -195,9 +221,7 @@ void FitterBase::PrepareOutput() { oss2 << oss.str() << "/D"; outTree->Branch(oss.str().c_str(), &syst_llh[i], oss2.str().c_str()); } - } - else - { + } else { outTree->Branch("LogL", &logLCurr, "LogL/D"); outTree->Branch("accProb", &accProb, "accProb/D"); outTree->Branch("step", &step, "step/i"); @@ -205,11 +229,11 @@ void FitterBase::PrepareOutput() { } MACH3LOG_INFO("-------------------- Starting MCMC --------------------"); - #ifdef DEBUG +#ifdef DEBUG if (debug) { debugFile << "----- Starting MCMC -----" << std::endl; } - #endif +#endif // Time the progress clock->Start(); @@ -218,7 +242,7 @@ void FitterBase::PrepareOutput() { // ******************* void FitterBase::SanitiseInputs() { -// ******************* + // ******************* for (size_t i = 0; i < samples.size(); ++i) { samples[i]->CleanMemoryBeforeFit(); } @@ -226,23 +250,29 @@ void FitterBase::SanitiseInputs() { // ******************* void FitterBase::SaveOutput() { -// ******************* - if(FileSaved) return; - //Stop Clock + // ******************* + if (FileSaved) + return; + // Stop Clock clock->Stop(); outputFile->cd(); outTree->Write(); - MACH3LOG_INFO("{} steps took {:.2e} seconds to complete. ({:.2e}s / step).", step - stepStart, clock->RealTime(), clock->RealTime() / static_cast(step - stepStart)); + MACH3LOG_INFO("{} steps took {:.2e} seconds to complete. ({:.2e}s / step).", + step - stepStart, clock->RealTime(), + clock->RealTime() / static_cast(step - stepStart)); MACH3LOG_INFO("{} steps were accepted.", accCount); - #ifdef DEBUG - if (debug) - { - debugFile << "\n\n" << step << " steps took " << clock->RealTime() << " seconds to complete. (" << clock->RealTime() / step << "s / step).\n" << accCount<< " steps were accepted." << std::endl; +#ifdef DEBUG + if (debug) { + debugFile << "\n\n" + << step << " steps took " << clock->RealTime() + << " seconds to complete. (" << clock->RealTime() / step + << "s / step).\n" + << accCount << " steps were accepted." << std::endl; debugFile.close(); } - #endif +#endif outputFile->Close(); FileSaved = true; @@ -250,17 +280,17 @@ void FitterBase::SaveOutput() { // ************************* // Add SampleHandler object to the Markov Chain -void FitterBase::AddSampleHandler(SampleHandlerBase * const sample) { -// ************************* +void FitterBase::AddSampleHandler(SampleHandlerBase *const sample) { + // ************************* // Check if any subsample name collides with already-registered subsamples for (const auto &s : samples) { for (int iExisting = 0; iExisting < s->GetNsamples(); ++iExisting) { for (int iNew = 0; iNew < sample->GetNsamples(); ++iNew) { if (s->GetSampleTitle(iExisting) == sample->GetSampleTitle(iNew)) { - MACH3LOG_ERROR( - "Duplicate sample title '{}' in handler {} detected: " - "same title exist in handler ", sample->GetSampleTitle(iNew), - sample->GetName(), s->GetName()); + MACH3LOG_ERROR("Duplicate sample title '{}' in handler {} detected: " + "same title exist in handler ", + sample->GetSampleTitle(iNew), sample->GetName(), + s->GetName()); throw MaCh3Exception(__FILE__, __LINE__); } } @@ -271,7 +301,6 @@ void FitterBase::AddSampleHandler(SampleHandlerBase * const sample) { if (s->GetName() == sample->GetName()) { MACH3LOG_WARN("SampleHandler with name '{}' already exists!", sample->GetName()); MACH3LOG_WARN("Is it intended?"); - //throw MaCh3Exception(__FILE__ , __LINE__ ); } } // Save additional info from samples @@ -279,7 +308,8 @@ void FitterBase::AddSampleHandler(SampleHandlerBase * const sample) { sample->SaveAdditionalInfo(SampleFolder); TotalNSamples += sample->GetNsamples(); - MACH3LOG_INFO("Adding {} object, with {} samples", sample->GetName(), sample->GetNsamples()); + MACH3LOG_INFO("Adding {} object, with {} samples", sample->GetName(), + sample->GetNsamples()); samples.push_back(sample); outputFile->cd(); } @@ -288,13 +318,13 @@ void FitterBase::AddSampleHandler(SampleHandlerBase * const sample) { // Add flux systematics, cross-section systematics, ND systematics to the chain void FitterBase::AddSystObj(ParameterHandlerBase * const cov) { // ************************* - MACH3LOG_INFO("Adding systematic object {}, with {} params", cov->GetName(), cov->GetNumParams()); + MACH3LOG_INFO("Adding systematic object {}, with {} params", cov->GetName(), cov->GetNumSystematicParams()); // KS: Need to make sure we don't have params with same name, otherwise ROOT I/O and parts of MaCh3 will be terribly confused... for (size_t s = 0; s < systematics.size(); ++s) { - for (int iPar = 0; iPar < systematics[s]->GetNumParams(); ++iPar) + for (int iPar = 0; iPar < systematics[s]->GetNumSystematicParams(); ++iPar) { - for (int i = 0; i < cov->GetNumParams(); ++i) + for (int i = 0; i < cov->GetNumSystematicParams(); ++i) { if(systematics[s]->GetParName(iPar) == cov->GetParName(i)){ MACH3LOG_ERROR("ParameterHandler {} has param '{}' which already exists in in {}, with name {}", @@ -314,21 +344,21 @@ void FitterBase::AddSystObj(ParameterHandlerBase * const cov) { systematics.push_back(cov); CovFolder->cd(); - std::vector n_vec(cov->GetNumParams()); - for (int i = 0; i < cov->GetNumParams(); ++i) { + std::vector n_vec(cov->GetNumSystematicParams()); + for (int i = 0; i < cov->GetNumSystematicParams(); ++i) { n_vec[i] = cov->GetParInit(i); } cov->GetCovMatrix()->Write(cov->GetName().c_str()); - TH2D* CorrMatrix = cov->GetCorrelationMatrix(); + TH2D *CorrMatrix = cov->GetCorrelationMatrix(); CorrMatrix->Write((cov->GetName() + std::string("_Corr")).c_str()); delete CorrMatrix; // If we have yaml config file for covariance let's save it YAML::Node Config = cov->GetConfig(); - if(!Config.IsNull()) - { - TMacro ConfigSave = YAMLtoTMacro(Config, (std::string("Config_") + cov->GetName())); + if (!Config.IsNull()) { + TMacro ConfigSave = + YAMLtoTMacro(Config, (std::string("Config_") + cov->GetName())); ConfigSave.Write(); } @@ -336,33 +366,34 @@ void FitterBase::AddSystObj(ParameterHandlerBase * const cov) { } // ******************* -void FitterBase::StartFromPreviousFit(const std::string& FitName) { -// ******************* +void FitterBase::StartFromPreviousFit(const std::string &FitName) { + // ******************* MACH3LOG_INFO("Getting starting position from {}", FitName); TFile *infile = M3::Open(FitName, "READ", __FILE__, __LINE__); TTree *posts = infile->Get("posteriors"); int step_val = 0; double log_val = M3::_LARGE_LOGL_; - posts->SetBranchAddress("step",&step_val); - posts->SetBranchAddress("LogL",&log_val); + posts->SetBranchAddress("step", &step_val); + posts->SetBranchAddress("LogL", &log_val); - for (size_t s = 0; s < systematics.size(); ++s) - { - TDirectory* CovarianceFolder = infile->Get("CovarianceFolder"); + for (size_t s = 0; s < systematics.size(); ++s) { + TDirectory *CovarianceFolder = infile->Get("CovarianceFolder"); std::string ConfigName = "Config_" + systematics[s]->GetName(); TMacro *ConfigCov = CovarianceFolder->Get(ConfigName.c_str()); - // KS: Not every covariance uses yaml, if it uses yaml make sure they are identical + // KS: Not every covariance uses yaml, if it uses yaml make sure they are + // identical if (ConfigCov != nullptr) { // Config which was in MCMC from which we are starting YAML::Node CovSettings = TMacroToYAML(*ConfigCov); // Config from currently used cov object YAML::Node ConfigCurrent = systematics[s]->GetConfig(); - if (!compareYAMLNodes(CovSettings, ConfigCurrent)) - { - MACH3LOG_ERROR("Yaml configs in previous chain (from path {}) and current one are different", FitName); - throw MaCh3Exception(__FILE__ , __LINE__ ); + if (!compareYAMLNodes(CovSettings, ConfigCurrent)) { + MACH3LOG_ERROR("Yaml configs in previous chain (from path {}) and " + "current one are different", + FitName); + throw MaCh3Exception(__FILE__, __LINE__); } delete ConfigCov; @@ -374,16 +405,18 @@ void FitterBase::StartFromPreviousFit(const std::string& FitName) { std::vector branch_vals; std::vector branch_name; systematics[s]->MatchMaCh3OutputBranches(posts, branch_vals, branch_name); - posts->GetEntry(posts->GetEntries()-1); + posts->GetEntry(posts->GetEntries() - 1); systematics[s]->SetParameters(branch_vals); systematics[s]->AcceptStep(); - MACH3LOG_INFO("Printing new starting values for: {}", systematics[s]->GetName()); + MACH3LOG_INFO("Printing new starting values for: {}", + systematics[s]->GetName()); systematics[s]->PrintNominalCurrProp(); - // Resetting branch addressed to nullptr as we don't want to write into a delected vector out of scope... - for (int i = 0; i < systematics[s]->GetNumParams(); ++i) { + // Resetting branch addressed to nullptr as we don't want to write into a + // delected vector out of scope... + for (int i = 0; i < systematics[s]->GetNumSystematicParams(); ++i) { posts->SetBranchAddress(systematics[s]->GetParName(i).c_str(), nullptr); } } @@ -394,7 +427,7 @@ void FitterBase::StartFromPreviousFit(const std::string& FitName) { delete infile; for (size_t s = 0; s < systematics.size(); ++s) { - if(systematics[s]->GetDoAdaption()){ //Use separate throw matrix for xsec + if (systematics[s]->GetDoAdaption()) { // Use separate throw matrix for xsec systematics[s]->SetNumberOfSteps(step_val); } } @@ -403,11 +436,13 @@ void FitterBase::StartFromPreviousFit(const std::string& FitName) { // ******************* // Process the MCMC output to get postfit etc void FitterBase::ProcessMCMC() { -// ******************* - if (fitMan == nullptr) return; + // ******************* + if (fitMan == nullptr) + return; // Process the MCMC - if (GetFromManager(fitMan->raw()["General"]["ProcessMCMC"], false, __FILE__ , __LINE__)){ + if (GetFromManager(fitMan->raw()["General"]["ProcessMCMC"], false, + __FILE__, __LINE__)) { // Make the processor MCMCProcessor Processor(std::string(outputFile->GetName())); @@ -436,7 +471,11 @@ void FitterBase::ProcessMCMC() { // Re-open the TFile if (!outputFile->IsOpen()) { MACH3LOG_INFO("Opening output again to update with means.."); - outputFile = new TFile(Get(fitMan->raw()["General"]["Output"]["Filename"], __FILE__, __LINE__).c_str(), "UPDATE"); + outputFile = new TFile( + Get(fitMan->raw()["General"]["Output"]["Filename"], + __FILE__, __LINE__) + .c_str(), + "UPDATE"); } Central->Write("PDF_Means"); Errors->Write("PDF_Errors"); @@ -450,35 +489,38 @@ void FitterBase::ProcessMCMC() { // ************************* // Run Drag Race void FitterBase::DragRace(const int NLaps) { -// ************************* + // ************************* MACH3LOG_INFO("Let the Race Begin!"); - MACH3LOG_INFO("All tests will be performed with {} threads", M3::GetNThreads()); + MACH3LOG_INFO("All tests will be performed with {} threads", + M3::GetNThreads()); // Reweight the MC - for(unsigned int ivs = 0; ivs < samples.size(); ++ivs) - { + for (unsigned int ivs = 0; ivs < samples.size(); ++ivs) { TStopwatch clockRace; clockRace.Start(); - for(int Lap = 0; Lap < NLaps; ++Lap) { + for (int Lap = 0; Lap < NLaps; ++Lap) { samples[ivs]->Reweight(); } clockRace.Stop(); - MACH3LOG_INFO("It took {:.4f} s to reweights {} times sample: {}", clockRace.RealTime(), NLaps, samples[ivs]->GetName()); - MACH3LOG_INFO("On average {:.6f}", clockRace.RealTime()/NLaps); + MACH3LOG_INFO("It took {:.4f} s to reweights {} times sample: {}", + clockRace.RealTime(), NLaps, samples[ivs]->GetName()); + MACH3LOG_INFO("On average {:.6f}", clockRace.RealTime() / NLaps); } - for(unsigned int ivs = 0; ivs < samples.size(); ++ivs) - { + for (unsigned int ivs = 0; ivs < samples.size(); ++ivs) { TStopwatch clockRace; clockRace.Start(); - for(int Lap = 0; Lap < NLaps; ++Lap) { + for (int Lap = 0; Lap < NLaps; ++Lap) { samples[ivs]->GetLikelihood(); } clockRace.Stop(); - MACH3LOG_INFO("It took {:.4f} s to calculate GetLikelihood {} times sample: {}", clockRace.RealTime(), NLaps, samples[ivs]->GetName()); - MACH3LOG_INFO("On average {:.6f}", clockRace.RealTime()/NLaps); + MACH3LOG_INFO( + "It took {:.4f} s to calculate GetLikelihood {} times sample: {}", + clockRace.RealTime(), NLaps, samples[ivs]->GetName()); + MACH3LOG_INFO("On average {:.6f}", clockRace.RealTime() / NLaps); } - // Get vector of proposed steps. If we want to run LLH scan or something else after we need to revert changes after proposing steps multiple times + // Get vector of proposed steps. If we want to run LLH scan or something else + // after we need to revert changes after proposing steps multiple times std::vector> StepsValuesBefore(systematics.size()); for (size_t s = 0; s < systematics.size(); ++s) { StepsValuesBefore[s] = systematics[s]->GetProposed(); @@ -486,12 +528,13 @@ void FitterBase::DragRace(const int NLaps) { for (size_t s = 0; s < systematics.size(); ++s) { TStopwatch clockRace; clockRace.Start(); - for(int Lap = 0; Lap < NLaps; ++Lap) { + for (int Lap = 0; Lap < NLaps; ++Lap) { systematics[s]->ProposeStep(); } clockRace.Stop(); - MACH3LOG_INFO("It took {:.4f} s to propose step {} times cov: {}", clockRace.RealTime(), NLaps, systematics[s]->GetName()); - MACH3LOG_INFO("On average {:.6f}", clockRace.RealTime()/NLaps); + MACH3LOG_INFO("It took {:.4f} s to propose step {} times cov: {}", + clockRace.RealTime(), NLaps, systematics[s]->GetName()); + MACH3LOG_INFO("On average {:.6f}", clockRace.RealTime() / NLaps); } for (size_t s = 0; s < systematics.size(); ++s) { systematics[s]->SetParameters(StepsValuesBefore[s]); @@ -500,22 +543,26 @@ void FitterBase::DragRace(const int NLaps) { for (size_t s = 0; s < systematics.size(); ++s) { TStopwatch clockRace; clockRace.Start(); - for(int Lap = 0; Lap < NLaps; ++Lap) { + for (int Lap = 0; Lap < NLaps; ++Lap) { systematics[s]->GetLikelihood(); } clockRace.Stop(); - MACH3LOG_INFO("It took {:.4f} s to calculate get likelihood {} times cov: {}", clockRace.RealTime(), NLaps, systematics[s]->GetName()); - MACH3LOG_INFO("On average {:.6f}", clockRace.RealTime()/NLaps); + MACH3LOG_INFO( + "It took {:.4f} s to calculate get likelihood {} times cov: {}", + clockRace.RealTime(), NLaps, systematics[s]->GetName()); + MACH3LOG_INFO("On average {:.6f}", clockRace.RealTime() / NLaps); } MACH3LOG_INFO("End of race"); } // ************************* -bool FitterBase::GetScanRange(std::map>& scanRanges) const { -// ************************* +bool FitterBase::GetScanRange( + std::map> &scanRanges) const { + // ************************* bool isScanRanges = false; - // YSP: Set up a mapping to store parameters with user-specified ranges, suggested by D. Barrow - if(fitMan->raw()["LLHScan"]["ScanRanges"]){ + // YSP: Set up a mapping to store parameters with user-specified ranges, + // suggested by D. Barrow + if (fitMan->raw()["LLHScan"]["ScanRanges"]) { YAML::Node scanRangesList = fitMan->raw()["LLHScan"]["ScanRanges"]; for (auto it = scanRangesList.begin(); it != scanRangesList.end(); ++it) { std::string itname = it->first.as(); @@ -525,19 +572,19 @@ bool FitterBase::GetScanRange(std::map>& scanRa } isScanRanges = true; } else { - MACH3LOG_INFO("There are no user-defined parameter ranges, so I'll use default param bounds for LLH Scans"); + MACH3LOG_INFO("There are no user-defined parameter ranges, so I'll use " + "default param bounds for LLH Scans"); } return isScanRanges; } // ************************* -bool FitterBase::CheckSkipParameter(const std::vector& SkipVector, const std::string& ParamName) const { -// ************************* +bool FitterBase::CheckSkipParameter(const std::vector &SkipVector, + const std::string &ParamName) const { + // ************************* bool skip = false; - for(unsigned int is = 0; is < SkipVector.size(); ++is) - { - if(ParamName.substr(0, SkipVector[is].length()) == SkipVector[is]) - { + for (unsigned int is = 0; is < SkipVector.size(); ++is) { + if (ParamName.substr(0, SkipVector[is].length()) == SkipVector[is]) { skip = true; break; } @@ -548,29 +595,32 @@ bool FitterBase::CheckSkipParameter(const std::vector& SkipVector, // ************************* // Run LLH scan void FitterBase::RunLLHScan() { -// ************************* + // ************************* // Save the settings into the output file SaveSettings(); MACH3LOG_INFO("Starting LLH Scan"); - //KS: Turn it on if you want LLH scan for each ND sample separately, which increase time significantly but can be useful for validating new samples or dials. - bool PlotLLHScanBySample = GetFromManager(fitMan->raw()["LLHScan"]["LLHScanBySample"], false, __FILE__ , __LINE__); - auto SkipVector = GetFromManager>(fitMan->raw()["LLHScan"]["LLHScanSkipVector"], {}, __FILE__ , __LINE__); + // KS: Turn it on if you want LLH scan for each ND sample separately, which + // increase time significantly but can be useful for validating new samples or + // dials. + bool PlotLLHScanBySample = GetFromManager( + fitMan->raw()["LLHScan"]["LLHScanBySample"], false, __FILE__, __LINE__); + auto SkipVector = GetFromManager>( + fitMan->raw()["LLHScan"]["LLHScanSkipVector"], {}, __FILE__, __LINE__); // Now finally get onto the LLH scan stuff - // Very similar code to MCMC but never start MCMC; just scan over the parameter space + // Very similar code to MCMC but never start MCMC; just scan over the + // parameter space std::vector Cov_LLH(systematics.size()); - for(unsigned int ivc = 0; ivc < systematics.size(); ++ivc ) - { + for (unsigned int ivc = 0; ivc < systematics.size(); ++ivc) { std::string NameTemp = systematics[ivc]->GetName(); NameTemp = NameTemp.substr(0, NameTemp.find("_cov")) + "_LLH"; Cov_LLH[ivc] = outputFile->mkdir(NameTemp.c_str()); } std::vector SampleClass_LLH(samples.size()); - for(unsigned int ivs = 0; ivs < samples.size(); ++ivs ) - { + for (unsigned int ivs = 0; ivs < samples.size(); ++ivs) { std::string NameTemp = samples[ivs]->GetName(); SampleClass_LLH[ivs] = outputFile->mkdir(NameTemp.c_str()); } @@ -578,218 +628,213 @@ void FitterBase::RunLLHScan() { TDirectory *Sample_LLH = outputFile->mkdir("Sample_LLH"); TDirectory *Total_LLH = outputFile->mkdir("Total_LLH"); - std::vectorSampleSplit_LLH; - if(PlotLLHScanBySample) - { + std::vector SampleSplit_LLH; + if (PlotLLHScanBySample) { SampleSplit_LLH.resize(TotalNSamples); int SampleIterator = 0; - for(unsigned int ivs = 0; ivs < samples.size(); ++ivs ) - { - for(int is = 0; is < samples[ivs]->GetNsamples(); ++is ) - { - SampleSplit_LLH[SampleIterator] = outputFile->mkdir((samples[ivs]->GetSampleTitle(is)+ "_LLH").c_str()); + for (unsigned int ivs = 0; ivs < samples.size(); ++ivs) { + for (int is = 0; is < samples[ivs]->GetNsamples(); ++is) { + SampleSplit_LLH[SampleIterator] = outputFile->mkdir( + (samples[ivs]->GetSampleTitle(is) + "_LLH").c_str()); SampleIterator++; } } } // Number of points we do for each LLH scan - const int n_points = GetFromManager(fitMan->raw()["LLHScan"]["LLHScanPoints"], 100, __FILE__ , __LINE__); - double nSigma = GetFromManager(fitMan->raw()["LLHScan"]["LLHScanSigma"], 1., __FILE__, __LINE__); + const int n_points = GetFromManager( + fitMan->raw()["LLHScan"]["LLHScanPoints"], 100, __FILE__, __LINE__); + double nSigma = GetFromManager(fitMan->raw()["LLHScan"]["LLHScanSigma"], + 1., __FILE__, __LINE__); // We print 5 reweights - const int countwidth = int(double(n_points)/double(5)); + const int countwidth = int(double(n_points) / double(5)); - // YSP: Set up a mapping to store parameters with user-specified ranges, suggested by D. Barrow + // YSP: Set up a mapping to store parameters with user-specified ranges, + // suggested by D. Barrow std::map> scanRanges; const bool isScanRanges = GetScanRange(scanRanges); - + // Loop over the covariance classes - for (ParameterHandlerBase *cov : systematics) - { + for (ParameterHandlerBase *cov : systematics) { // Scan over all the parameters // Get the number of parameters - int npars = cov->GetNumParams(); - bool IsPCA = cov->IsPCA(); - if (IsPCA) npars = cov->GetNParameters(); - for (int i = 0; i < npars; ++i) - { + int npars = cov->GetNumSystematicParams(); + for (int i = 0; i < npars; ++i) { // Get the parameter name std::string name = cov->GetParFancyName(i); - if (IsPCA) name += "_PCA"; // KS: Check if we want to skip this parameter - if(CheckSkipParameter(SkipVector, name)) continue; + if (CheckSkipParameter(SkipVector, name)) + continue; // Get the parameter priors and bounds double prior = cov->GetParInit(i); - if (IsPCA) prior = cov->GetPCAHandler()->GetParCurrPCA(i); // Get the covariance matrix and do the +/- nSigma // Set lower and upper bounds relative the prior // Set the parameter ranges between which LLH points are scanned - double lower = prior - nSigma*cov->GetDiagonalError(i); - double upper = prior + nSigma*cov->GetDiagonalError(i); - // If PCA, transform these parameter values to the PCA basis - if (IsPCA) { - lower = prior - nSigma*std::sqrt((cov->GetPCAHandler()->GetEigenValues())(i)); - upper = prior + nSigma*std::sqrt((cov->GetPCAHandler()->GetEigenValues())(i)); - MACH3LOG_INFO("eval {} = {:.2f}", i, cov->GetPCAHandler()->GetEigenValues()(i)); - MACH3LOG_INFO("prior {} = {:.2f}", i, prior); - MACH3LOG_INFO("lower {} = {:.2f}", i, lower); - MACH3LOG_INFO("upper {} = {:.2f}", i, upper); - MACH3LOG_INFO("nSigma = {:.2f}", nSigma); - } - // Implementation suggested by D. Barrow - // If param ranges are specified in scanRanges node, extract it from there - if(isScanRanges){ + double lower = prior - nSigma * cov->GetDiagonalError(i); + double upper = prior + nSigma * cov->GetDiagonalError(i); + + // Implementation suggested by D. Barrow + // If param ranges are specified in scanRanges node, extract it from there + if (isScanRanges) { // Find matching entries through std::maps auto it = scanRanges.find(name); - if (it != scanRanges.end() && it->second.size() == 2) { //Making sure the range is has only two entries + if (it != scanRanges.end() && + it->second.size() == + 2) { // Making sure the range is has only two entries lower = it->second[0]; upper = it->second[1]; - MACH3LOG_INFO("Found matching param name for setting specified range for {}", name); + MACH3LOG_INFO( + "Found matching param name for setting specified range for {}", + name); MACH3LOG_INFO("Range for {} = [{:.2f}, {:.2f}]", name, lower, upper); } } - - // Cross-section and flux parameters have boundaries that we scan between, check that these are respected in setting lower and upper variables + + // Cross-section and flux parameters have boundaries that we scan between, + // check that these are respected in setting lower and upper variables // This also applies for other parameters like osc, etc. lower = std::max(lower, cov->GetLowerBound(i)); upper = std::min(upper, cov->GetUpperBound(i)); - MACH3LOG_INFO("Scanning {} with {} steps, from [{:.2f} , {:.2f}], prior = {:.2f}", name, n_points, lower, upper, prior); + MACH3LOG_INFO( + "Scanning {} with {} steps, from [{:.2f} , {:.2f}], prior = {:.2f}", + name, n_points, lower, upper, prior); // Make the TH1D - auto hScan = std::make_unique((name + "_full").c_str(), (name + "_full").c_str(), n_points, lower, upper); - hScan->SetTitle((std::string("2LLH_full, ") + name + ";" + name + "; -2(ln L_{sample} + ln L_{xsec+flux} + ln L_{det})").c_str()); + auto hScan = std::make_unique((name + "_full").c_str(), + (name + "_full").c_str(), n_points, + lower, upper); + hScan->SetTitle((std::string("2LLH_full, ") + name + ";" + name + + "; -2(ln L_{sample} + ln L_{xsec+flux} + ln L_{det})") + .c_str()); hScan->SetDirectory(nullptr); - auto hScanSam = std::make_unique((name + "_sam").c_str(), (name + "_sam").c_str(), n_points, lower, upper); - hScanSam->SetTitle((std::string("2LLH_sam, ") + name + ";" + name + "; -2(ln L_{sample})").c_str()); + auto hScanSam = std::make_unique((name + "_sam").c_str(), + (name + "_sam").c_str(), n_points, + lower, upper); + hScanSam->SetTitle((std::string("2LLH_sam, ") + name + ";" + name + + "; -2(ln L_{sample})") + .c_str()); hScanSam->SetDirectory(nullptr); std::vector> hScanSample(samples.size()); std::vector nSamLLH(samples.size()); - for(unsigned int ivs = 0; ivs < samples.size(); ++ivs ) - { + for (unsigned int ivs = 0; ivs < samples.size(); ++ivs) { std::string NameTemp = samples[ivs]->GetName(); - hScanSample[ivs] = std::make_unique((name+"_"+NameTemp).c_str(), (name+"_" + NameTemp).c_str(), n_points, lower, upper); + hScanSample[ivs] = std::make_unique( + (name + "_" + NameTemp).c_str(), (name + "_" + NameTemp).c_str(), + n_points, lower, upper); hScanSample[ivs]->SetDirectory(nullptr); - hScanSample[ivs]->SetTitle(("2LLH_" + NameTemp + ", " + name + ";" + name + "; -2(ln L_{" + NameTemp +"})").c_str()); + hScanSample[ivs]->SetTitle(("2LLH_" + NameTemp + ", " + name + ";" + + name + "; -2(ln L_{" + NameTemp + "})") + .c_str()); nSamLLH[ivs] = 0.; } std::vector> hScanCov(systematics.size()); std::vector nCovLLH(systematics.size()); - for(unsigned int ivc = 0; ivc < systematics.size(); ++ivc ) - { + for (unsigned int ivc = 0; ivc < systematics.size(); ++ivc) { std::string NameTemp = systematics[ivc]->GetName(); NameTemp = NameTemp.substr(0, NameTemp.find("_cov")); - hScanCov[ivc] = std::make_unique((name + "_" + NameTemp).c_str(), (name + "_" + NameTemp).c_str(), n_points, lower, upper); + hScanCov[ivc] = std::make_unique((name + "_" + NameTemp).c_str(), + (name + "_" + NameTemp).c_str(), + n_points, lower, upper); hScanCov[ivc]->SetDirectory(nullptr); - hScanCov[ivc]->SetTitle(("2LLH_" + NameTemp + ", " + name + ";" + name + "; -2(ln L_{" + NameTemp +"})").c_str()); + hScanCov[ivc]->SetTitle(("2LLH_" + NameTemp + ", " + name + ";" + name + + "; -2(ln L_{" + NameTemp + "})") + .c_str()); nCovLLH[ivc] = 0.; } - std::vector hScanSamSplit; + std::vector hScanSamSplit; std::vector sampleSplitllh; - if(PlotLLHScanBySample) - { + if (PlotLLHScanBySample) { int SampleIterator = 0; - for(unsigned int ivs = 0; ivs < samples.size(); ++ivs ) - { + for (unsigned int ivs = 0; ivs < samples.size(); ++ivs) { hScanSamSplit.resize(TotalNSamples); sampleSplitllh.resize(TotalNSamples); - for(int is = 0; is < samples[ivs]->GetNsamples(); ++is ) - { - hScanSamSplit[SampleIterator] = new TH1D((name+samples[ivs]->GetSampleTitle(is)).c_str(), (name+samples[ivs]->GetSampleTitle(is)).c_str(), n_points, lower, upper); - hScanSamSplit[SampleIterator]->SetTitle((std::string("2LLH_sam, ") + name + ";" + name + "; -2(ln L_{sample})").c_str()); + for (int is = 0; is < samples[ivs]->GetNsamples(); ++is) { + hScanSamSplit[SampleIterator] = + new TH1D((name + samples[ivs]->GetSampleTitle(is)).c_str(), + (name + samples[ivs]->GetSampleTitle(is)).c_str(), + n_points, lower, upper); + hScanSamSplit[SampleIterator]->SetTitle((std::string("2LLH_sam, ") + + name + ";" + name + + "; -2(ln L_{sample})") + .c_str()); SampleIterator++; } } } // Scan over the parameter space - for (int j = 0; j < n_points; ++j) - { + for (int j = 0; j < n_points; ++j) { if (j % countwidth == 0) MaCh3Utils::PrintProgressBar(j, n_points); - // For PCA we have to do it differently - if (IsPCA) { - cov->GetPCAHandler()->SetParPropPCA(i, hScan->GetBinCenter(j+1)); - } else { - // Set the parameter - cov->SetParProp(i, hScan->GetBinCenter(j+1)); - } + cov->SetParProp(i, hScan->GetBinCenter(j + 1)); // Reweight the MC - for(unsigned int ivs = 0; ivs < samples.size(); ++ivs ) - { + for (unsigned int ivs = 0; ivs < samples.size(); ++ivs) { samples[ivs]->Reweight(); } - //Total LLH + // Total LLH double totalllh = 0; // Get the -log L likelihoods double samplellh = 0; - for(unsigned int ivs = 0; ivs < samples.size(); ++ivs ) - { + for (unsigned int ivs = 0; ivs < samples.size(); ++ivs) { nSamLLH[ivs] = samples[ivs]->GetLikelihood(); samplellh += nSamLLH[ivs]; } - for(unsigned int ivc = 0; ivc < systematics.size(); ++ivc ) - { + for (unsigned int ivc = 0; ivc < systematics.size(); ++ivc) { nCovLLH[ivc] = systematics[ivc]->GetLikelihood(); totalllh += nCovLLH[ivc]; } totalllh += samplellh; - if(PlotLLHScanBySample) - { + if (PlotLLHScanBySample) { int SampleIterator = 0; - for(unsigned int ivs = 0; ivs < samples.size(); ++ivs ) - { - for(int is = 0; is < samples[ivs]->GetNsamples(); ++is) - { - sampleSplitllh[SampleIterator] = samples[ivs]->GetSampleLikelihood(is); + for (unsigned int ivs = 0; ivs < samples.size(); ++ivs) { + for (int is = 0; is < samples[ivs]->GetNsamples(); ++is) { + sampleSplitllh[SampleIterator] = + samples[ivs]->GetSampleLikelihood(is); SampleIterator++; } } } - for(unsigned int ivs = 0; ivs < samples.size(); ++ivs ) { - hScanSample[ivs]->SetBinContent(j+1, 2*nSamLLH[ivs]); + for (unsigned int ivs = 0; ivs < samples.size(); ++ivs) { + hScanSample[ivs]->SetBinContent(j + 1, 2 * nSamLLH[ivs]); } - for(unsigned int ivc = 0; ivc < systematics.size(); ++ivc ) { - hScanCov[ivc]->SetBinContent(j+1, 2*nCovLLH[ivc]); + for (unsigned int ivc = 0; ivc < systematics.size(); ++ivc) { + hScanCov[ivc]->SetBinContent(j + 1, 2 * nCovLLH[ivc]); } - hScanSam->SetBinContent(j+1, 2*samplellh); - hScan->SetBinContent(j+1, 2*totalllh); + hScanSam->SetBinContent(j + 1, 2 * samplellh); + hScan->SetBinContent(j + 1, 2 * totalllh); - if(PlotLLHScanBySample) - { + if (PlotLLHScanBySample) { int SampleIterator = 0; - for(unsigned int ivs = 0; ivs < samples.size(); ++ivs ) - { - for(int is = 0; is < samples[ivs]->GetNsamples(); ++is) - { - hScanSamSplit[SampleIterator]->SetBinContent(j+1, 2*sampleSplitllh[SampleIterator]); + for (unsigned int ivs = 0; ivs < samples.size(); ++ivs) { + for (int is = 0; is < samples[ivs]->GetNsamples(); ++is) { + hScanSamSplit[SampleIterator]->SetBinContent( + j + 1, 2 * sampleSplitllh[SampleIterator]); SampleIterator++; } } } } - for(unsigned int ivc = 0; ivc < systematics.size(); ++ivc ) - { + for (unsigned int ivc = 0; ivc < systematics.size(); ++ivc) { Cov_LLH[ivc]->cd(); hScanCov[ivc]->Write(); } - for(unsigned int ivs = 0; ivs < samples.size(); ++ivs ) - { + for (unsigned int ivs = 0; ivs < samples.size(); ++ivs) { SampleClass_LLH[ivs]->cd(); hScanSample[ivs]->Write(); } @@ -798,13 +843,10 @@ void FitterBase::RunLLHScan() { Total_LLH->cd(); hScan->Write(); - if(PlotLLHScanBySample) - { + if (PlotLLHScanBySample) { int SampleIterator = 0; - for(unsigned int ivs = 0; ivs < samples.size(); ++ivs ) - { - for(int is = 0; is < samples[ivs]->GetNsamples(); ++is) - { + for (unsigned int ivs = 0; ivs < samples.size(); ++ivs) { + for (int is = 0; is < samples[ivs]->GetNsamples(); ++is) { SampleSplit_LLH[SampleIterator]->cd(); hScanSamSplit[SampleIterator]->Write(); delete hScanSamSplit[SampleIterator]; @@ -814,22 +856,16 @@ void FitterBase::RunLLHScan() { } // Reset the parameters to their prior central values - if (IsPCA) { - cov->GetPCAHandler()->SetParPropPCA(i, prior); - } else { - cov->SetParProp(i, prior); - } - }//end loop over systematics - }//end loop covariance classes + cov->SetParProp(i, prior); + } // end loop over systematics + } // end loop covariance classes - for(unsigned int ivc = 0; ivc < systematics.size(); ++ivc ) - { + for (unsigned int ivc = 0; ivc < systematics.size(); ++ivc) { Cov_LLH[ivc]->Write(); delete Cov_LLH[ivc]; } - for(unsigned int ivs = 0; ivs < samples.size(); ++ivs ) - { + for (unsigned int ivs = 0; ivs < samples.size(); ++ivs) { SampleClass_LLH[ivs]->Write(); delete SampleClass_LLH[ivs]; } @@ -840,13 +876,10 @@ void FitterBase::RunLLHScan() { Total_LLH->Write(); delete Total_LLH; - if(PlotLLHScanBySample) - { + if (PlotLLHScanBySample) { int SampleIterator = 0; - for(unsigned int ivs = 0; ivs < samples.size(); ++ivs ) - { - for(int is = 0; is < samples[ivs]->GetNsamples(); ++is ) - { + for (unsigned int ivs = 0; ivs < samples.size(); ++ivs) { + for (int is = 0; is < samples[ivs]->GetNsamples(); ++is) { SampleSplit_LLH[SampleIterator]->Write(); delete SampleSplit_LLH[SampleIterator]; SampleIterator++; @@ -856,45 +889,49 @@ void FitterBase::RunLLHScan() { } // ************************* -//LLH scan is good first estimate of step scale +// LLH scan is good first estimate of step scale void FitterBase::GetStepScaleBasedOnLLHScan() { -// ************************* + // ************************* TDirectory *Sample_LLH = outputFile->Get("Sample_LLH"); MACH3LOG_INFO("Starting Get Step Scale Based On LLHScan"); - if(Sample_LLH->IsZombie()) - { - MACH3LOG_WARN("Couldn't find Sample_LLH, it looks like LLH scan wasn't run, will do this now"); + if (Sample_LLH->IsZombie()) { + MACH3LOG_WARN("Couldn't find Sample_LLH, it looks like LLH scan wasn't " + "run, will do this now"); RunLLHScan(); } - for (ParameterHandlerBase *cov : systematics) - { - const int npars = cov->GetNumParams(); + for (ParameterHandlerBase *cov : systematics) { + const int npars = cov->GetNumSystematicParams(); std::vector StepScale(npars); - for (int i = 0; i < npars; ++i) - { + for (int i = 0; i < npars; ++i) { std::string name = cov->GetParFancyName(i); StepScale[i] = cov->GetIndivStepScale(i); - TH1D* LLHScan = Sample_LLH->Get((name+"_sam").c_str()); - if(LLHScan == nullptr) - { + TH1D *LLHScan = Sample_LLH->Get((name + "_sam").c_str()); + if (LLHScan == nullptr) { MACH3LOG_WARN("Couldn't find LLH scan, for {}, skipping", name); continue; } - const double LLH_val = std::max(LLHScan->GetBinContent(1), LLHScan->GetBinContent(LLHScan->GetNbinsX())); - //If there is no sensitivity leave it - if(LLH_val < 0.001) continue; + const double LLH_val = + std::max(LLHScan->GetBinContent(1), + LLHScan->GetBinContent(LLHScan->GetNbinsX())); + // If there is no sensitivity leave it + if (LLH_val < 0.001) + continue; - // EM: assuming that the likelihood is gaussian, approximate sigma value is given by variation/sqrt(-2LLH) - // can evaluate this at any point, simple to evaluate it in the first bin of the LLH scan - // KS: We assume variation is 1 sigma, each dial has different scale so it becomes faff... + // EM: assuming that the likelihood is gaussian, approximate sigma value + // is given by variation/sqrt(-2LLH) can evaluate this at any point, + // simple to evaluate it in the first bin of the LLH scan KS: We assume + // variation is 1 sigma, each dial has different scale so it becomes + // faff... const double Var = 1.; - const double approxSigma = TMath::Abs(Var)/std::sqrt(LLH_val); + const double approxSigma = TMath::Abs(Var) / std::sqrt(LLH_val); - // Based on Ewan comment I just took the 1sigma width from the LLH, assuming it was Gaussian, but then had to also scale by 2.38/sqrt(N_params) - const double NewStepScale = approxSigma * 2.38/std::sqrt(npars); + // Based on Ewan comment I just took the 1sigma width from the LLH, + // assuming it was Gaussian, but then had to also scale + // by 2.38/sqrt(N_params) + const double NewStepScale = approxSigma * 2.38 / std::sqrt(npars); StepScale[i] = NewStepScale; MACH3LOG_DEBUG("Sigma: {}", approxSigma); MACH3LOG_DEBUG("optimal Step Size: {}", NewStepScale); @@ -907,175 +944,164 @@ void FitterBase::GetStepScaleBasedOnLLHScan() { // ************************* // Run 2D LLH scan void FitterBase::Run2DLLHScan() { -// ************************* + // ************************* // Save the settings into the output file SaveSettings(); MACH3LOG_INFO("Starting 2D LLH Scan"); TDirectory *Sample_2DLLH = outputFile->mkdir("Sample_2DLLH"); - auto SkipVector = GetFromManager>(fitMan->raw()["LLHScan"]["LLHScanSkipVector"], {}, __FILE__ , __LINE__);; + auto SkipVector = GetFromManager>( + fitMan->raw()["LLHScan"]["LLHScanSkipVector"], {}, __FILE__, __LINE__); + ; // Number of points we do for each LLH scan - const int n_points = GetFromManager(fitMan->raw()["LLHScan"]["2DLLHScanPoints"], 20, __FILE__ , __LINE__); + const int n_points = GetFromManager( + fitMan->raw()["LLHScan"]["2DLLHScanPoints"], 20, __FILE__, __LINE__); // We print 5 reweights - const int countwidth = int(double(n_points)/double(5)); + const int countwidth = int(double(n_points) / double(5)); std::map> scanRanges; const bool isScanRanges = GetScanRange(scanRanges); - const double nSigma = GetFromManager(fitMan->raw()["LLHScan"]["LLHScanSigma"], 1., __FILE__, __LINE__); + const double nSigma = GetFromManager( + fitMan->raw()["LLHScan"]["LLHScanSigma"], 1., __FILE__, __LINE__); // Loop over the covariance classes - for (ParameterHandlerBase *cov : systematics) - { + for (ParameterHandlerBase *cov : systematics) { // Scan over all the parameters // Get the number of parameters - int npars = cov->GetNumParams(); - bool IsPCA = cov->IsPCA(); - if (IsPCA) npars = cov->GetNParameters(); + int npars = cov->GetNumSystematicParams(); - for (int i = 0; i < npars; ++i) - { + for (int i = 0; i < npars; ++i) { std::string name_x = cov->GetParFancyName(i); - if (IsPCA) name_x += "_PCA"; // Get the parameter priors and bounds double prior_x = cov->GetParInit(i); - if (IsPCA) prior_x = cov->GetPCAHandler()->GetParCurrPCA(i); // Get the covariance matrix and do the +/- nSigma // Set lower and upper bounds relative the prior - double lower_x = prior_x - nSigma*cov->GetDiagonalError(i); - double upper_x = prior_x + nSigma*cov->GetDiagonalError(i); - // If PCA, transform these parameter values to the PCA basis - if (IsPCA) { - lower_x = prior_x - nSigma*std::sqrt((cov->GetPCAHandler()->GetEigenValues())(i)); - upper_x = prior_x + nSigma*std::sqrt((cov->GetPCAHandler()->GetEigenValues())(i)); - MACH3LOG_INFO("eval {} = {:.2f}", i, cov->GetPCAHandler()->GetEigenValues()(i)); - MACH3LOG_INFO("prior {} = {:.2f}", i, prior_x); - MACH3LOG_INFO("lower {} = {:.2f}", i, lower_x); - MACH3LOG_INFO("upper {} = {:.2f}", i, upper_x); - MACH3LOG_INFO("nSigma = {:.2f}", nSigma); - } + double lower_x = prior_x - nSigma * cov->GetDiagonalError(i); + double upper_x = prior_x + nSigma * cov->GetDiagonalError(i); + // If param ranges are specified in scanRanges node, extract it from there - if(isScanRanges){ + if (isScanRanges) { // Find matching entries through std::maps auto it = scanRanges.find(name_x); - if (it != scanRanges.end() && it->second.size() == 2) { //Making sure the range is has only two entries + if (it != scanRanges.end() && + it->second.size() == + 2) { // Making sure the range is has only two entries lower_x = it->second[0]; upper_x = it->second[1]; - MACH3LOG_INFO("Found matching param name for setting specified range for {}", name_x); - MACH3LOG_INFO("Range for {} = [{:.2f}, {:.2f}]", name_x, lower_x, upper_x); + MACH3LOG_INFO( + "Found matching param name for setting specified range for {}", + name_x); + MACH3LOG_INFO("Range for {} = [{:.2f}, {:.2f}]", name_x, lower_x, + upper_x); } } - // Cross-section and flux parameters have boundaries that we scan between, check that these are respected in setting lower and upper variables + // Cross-section and flux parameters have boundaries that we scan between, + // check that these are respected in setting lower and upper variables lower_x = std::max(lower_x, cov->GetLowerBound(i)); upper_x = std::min(upper_x, cov->GetUpperBound(i)); // KS: Check if we want to skip this parameter - if(CheckSkipParameter(SkipVector, name_x)) continue; + if (CheckSkipParameter(SkipVector, name_x)) + continue; - for (int j = 0; j < i; ++j) - { + for (int j = 0; j < i; ++j) { std::string name_y = cov->GetParFancyName(j); - if (IsPCA) name_y += "_PCA"; // KS: Check if we want to skip this parameter - if(CheckSkipParameter(SkipVector, name_y)) continue; + if (CheckSkipParameter(SkipVector, name_y)) + continue; // Get the parameter priors and bounds double prior_y = cov->GetParInit(j); - if (IsPCA) prior_y = cov->GetPCAHandler()->GetParCurrPCA(j); // Set lower and upper bounds relative the prior - double lower_y = prior_y - nSigma*cov->GetDiagonalError(j); - double upper_y = prior_y + nSigma*cov->GetDiagonalError(j); - // If PCA, transform these parameter values to the PCA basis - if (IsPCA) { - lower_y = prior_y - nSigma*std::sqrt((cov->GetPCAHandler()->GetEigenValues())(j)); - upper_y = prior_y + nSigma*std::sqrt((cov->GetPCAHandler()->GetEigenValues())(j)); - MACH3LOG_INFO("eval {} = {:.2f}", i, cov->GetPCAHandler()->GetEigenValues()(j)); - MACH3LOG_INFO("prior {} = {:.2f}", i, prior_y); - MACH3LOG_INFO("lower {} = {:.2f}", i, lower_y); - MACH3LOG_INFO("upper {} = {:.2f}", i, upper_y); - MACH3LOG_INFO("nSigma = {:.2f}", nSigma); - } - // If param ranges are specified in scanRanges node, extract it from there - if(isScanRanges){ + double lower_y = prior_y - nSigma * cov->GetDiagonalError(j); + double upper_y = prior_y + nSigma * cov->GetDiagonalError(j); + + // If param ranges are specified in scanRanges node, extract it from + // there + if (isScanRanges) { // Find matching entries through std::maps auto it = scanRanges.find(name_y); - if (it != scanRanges.end() && it->second.size() == 2) { //Making sure the range is has only two entries + if (it != scanRanges.end() && + it->second.size() == + 2) { // Making sure the range is has only two entries lower_y = it->second[0]; upper_y = it->second[1]; - MACH3LOG_INFO("Found matching param name for setting specified range for {}", name_y); - MACH3LOG_INFO("Range for {} = [{:.2f}, {:.2f}]", name_y, lower_y, upper_y); + MACH3LOG_INFO( + "Found matching param name for setting specified range for {}", + name_y); + MACH3LOG_INFO("Range for {} = [{:.2f}, {:.2f}]", name_y, lower_y, + upper_y); } } - // Cross-section and flux parameters have boundaries that we scan between, check that these are respected in setting lower and upper variables + // Cross-section and flux parameters have boundaries that we scan + // between, check that these are respected in setting lower and upper + // variables lower_y = std::max(lower_y, cov->GetLowerBound(j)); upper_y = std::min(upper_y, cov->GetUpperBound(j)); - MACH3LOG_INFO("Scanning X {} with {} steps, from {:.2f} - {:.2f}, prior = {}", name_x, n_points, lower_x, upper_x, prior_x); - MACH3LOG_INFO("Scanning Y {} with {} steps, from {:.2f} - {:.2f}, prior = {}", name_y, n_points, lower_y, upper_y, prior_y); - - auto hScanSam = std::make_unique((name_x + "_" + name_y + "_sam").c_str(), (name_x + "_" + name_y + "_sam").c_str(), - n_points, lower_x, upper_x, n_points, lower_y, upper_y); + MACH3LOG_INFO( + "Scanning X {} with {} steps, from {:.2f} - {:.2f}, prior = {}", + name_x, n_points, lower_x, upper_x, prior_x); + MACH3LOG_INFO( + "Scanning Y {} with {} steps, from {:.2f} - {:.2f}, prior = {}", + name_y, n_points, lower_y, upper_y, prior_y); + + auto hScanSam = std::make_unique( + (name_x + "_" + name_y + "_sam").c_str(), + (name_x + "_" + name_y + "_sam").c_str(), n_points, lower_x, + upper_x, n_points, lower_y, upper_y); hScanSam->SetDirectory(nullptr); hScanSam->GetXaxis()->SetTitle(name_x.c_str()); hScanSam->GetYaxis()->SetTitle(name_y.c_str()); hScanSam->GetZaxis()->SetTitle("2LLH_sam"); // Scan over the parameter space - for (int x = 0; x < n_points; ++x) - { + for (int x = 0; x < n_points; ++x) { if (x % countwidth == 0) MaCh3Utils::PrintProgressBar(x, n_points); - for (int y = 0; y < n_points; ++y) - { - // For PCA we have to do it differently - if (IsPCA) { - cov->GetPCAHandler()->SetParPropPCA(i, hScanSam->GetXaxis()->GetBinCenter(x+1)); - cov->GetPCAHandler()->SetParPropPCA(j, hScanSam->GetYaxis()->GetBinCenter(y+1)); - } else { - // Set the parameter - cov->SetParProp(i, hScanSam->GetXaxis()->GetBinCenter(x+1)); - cov->SetParProp(j, hScanSam->GetYaxis()->GetBinCenter(y+1)); - } + for (int y = 0; y < n_points; ++y) { + + // Set the parameter + cov->SetParProp(i, hScanSam->GetXaxis()->GetBinCenter(x + 1)); + cov->SetParProp(j, hScanSam->GetYaxis()->GetBinCenter(y + 1)); // Reweight the MC - for(unsigned int ivs = 0; ivs < samples.size(); ++ivs) { + for (unsigned int ivs = 0; ivs < samples.size(); ++ivs) { samples[ivs]->Reweight(); } // Get the -log L likelihoods double samplellh = 0; - for(unsigned int ivs = 0; ivs < samples.size(); ++ivs) { + for (unsigned int ivs = 0; ivs < samples.size(); ++ivs) { samplellh += samples[ivs]->GetLikelihood(); } - hScanSam->SetBinContent(x+1, y+1, 2*samplellh); - }// end loop over y points + hScanSam->SetBinContent(x + 1, y + 1, 2 * samplellh); + } // end loop over y points } // end loop over x points Sample_2DLLH->cd(); hScanSam->Write(); // Reset the parameters to their prior central values - if (IsPCA) { - cov->GetPCAHandler()->SetParPropPCA(i, prior_x); - cov->GetPCAHandler()->SetParPropPCA(j, prior_y); - } else { - cov->SetParProp(i, prior_x); - cov->SetParProp(j, prior_y); - } - } //end loop over systematics y - }//end loop over systematics X - }//end loop covariance classes + + cov->SetParProp(i, prior_x); + cov->SetParProp(j, prior_y); + + } // end loop over systematics y + } // end loop over systematics X + } // end loop covariance classes Sample_2DLLH->Write(); delete Sample_2DLLH; } // ************************* void FitterBase::RunSigmaVar() { -// ************************* + // ************************* // Save the settings into the output file SaveSettings(); @@ -1088,44 +1114,40 @@ void FitterBase::RunSigmaVar() { outputFile->cd(); - //KS: this is only relevant if PlotByMode is turned on - //Checking each mode is time consuming so we only consider one which are relevant for particular analysis + // KS: this is only relevant if PlotByMode is turned on + // Checking each mode is time consuming so we only consider one which are + // relevant for particular analysis constexpr int nRelevantModes = 2; - bool DoByMode = GetFromManager(fitMan->raw()["General"]["DoByMode"], false, __FILE__ , __LINE__); + bool DoByMode = GetFromManager(fitMan->raw()["General"]["DoByMode"], + false, __FILE__, __LINE__); - //KS: If true it will make additional plots with LLH sample contribution in each bin, should make it via config file... + // KS: If true it will make additional plots with LLH sample contribution in + // each bin, should make it via config file... bool PlotLLHperBin = false; - auto SkipVector = GetFromManager>(fitMan->raw()["LLHScan"]["LLHScanSkipVector"], {}, __FILE__ , __LINE__);; + auto SkipVector = GetFromManager>( + fitMan->raw()["LLHScan"]["LLHScanSkipVector"], {}, __FILE__, __LINE__); + ; - for (ParameterHandlerBase *cov : systematics) - { + for (ParameterHandlerBase *cov : systematics) { TMatrixDSym *Cov = cov->GetCovMatrix(); - if(cov->IsPCA()) - { - MACH3LOG_ERROR("Using PCAed matrix not implemented within sigma var code, I am sorry :("); - throw MaCh3Exception(__FILE__ , __LINE__ ); - } - // Loop over xsec parameters - for (int i = 0; i < cov->GetNumParams(); ++i) - { + for (int i = 0; i < cov->GetNumSystematicParams(); ++i) { // Get the parameter name std::string name = cov->GetParFancyName(i); // KS: Check if we want to skip this parameter - if(CheckSkipParameter(SkipVector, name)) continue; + if (CheckSkipParameter(SkipVector, name)) + continue; outputFile->cd(); - TDirectory* dirArryDial = outputFile->mkdir(name.c_str()); - std::vector dirArrySample(TotalNSamples); + TDirectory *dirArryDial = outputFile->mkdir(name.c_str()); + std::vector dirArrySample(TotalNSamples); int SampleIterator = 0; // Get each sample and how it's responded to our reweighted parameter - for(unsigned int ivs = 0; ivs < samples.size(); ivs++ ) - { - for(int k = 0; k < samples[ivs]->GetNsamples(); k++ ) - { + for (unsigned int ivs = 0; ivs < samples.size(); ivs++) { + for (int k = 0; k < samples[ivs]->GetNsamples(); k++) { std::string title = std::string(samples[ivs]->GetPDF(k)->GetName()); dirArryDial->cd(); dirArrySample[SampleIterator] = dirArryDial->mkdir(title.c_str()); @@ -1144,27 +1166,27 @@ void FitterBase::RunSigmaVar() { // Set up for single mode TH1D ****sigmaArray_mode_x = nullptr; TH1D ****sigmaArray_mode_y = nullptr; - if (DoByMode) - { - sigmaArray_mode_x = new TH1D***[numVar](); - sigmaArray_mode_y = new TH1D***[numVar](); + if (DoByMode) { + sigmaArray_mode_x = new TH1D ***[numVar](); + sigmaArray_mode_y = new TH1D ***[numVar](); } MACH3LOG_INFO("{:<20}{}", name, ""); // Make asymmetric errors just in case - for (int j = 0; j < numVar; ++j) - { + for (int j = 0; j < numVar; ++j) { // New value = prior + variation*1sigma uncertainty - double paramVal = cov->GetParInit(i)+sigmaArray[j]*std::sqrt((*Cov)(i,i)); + double paramVal = + cov->GetParInit(i) + sigmaArray[j] * std::sqrt((*Cov)(i, i)); // Check the bounds on the parameter - paramVal = std::max(cov->GetLowerBound(i), std::min(paramVal, cov->GetUpperBound(i))); + paramVal = std::max(cov->GetLowerBound(i), + std::min(paramVal, cov->GetUpperBound(i))); // Set the parameter cov->SetParProp(i, paramVal); // And reweight the sample - for(unsigned int ivs = 0; ivs < samples.size(); ivs++) { + for (unsigned int ivs = 0; ivs < samples.size(); ivs++) { samples[ivs]->Reweight(); } @@ -1173,95 +1195,120 @@ void FitterBase::RunSigmaVar() { sigmaArray_x_norm[j].resize(TotalNSamples); sigmaArray_y_norm[j].resize(TotalNSamples); - if (DoByMode) - { - sigmaArray_mode_x[j] = new TH1D**[TotalNSamples](); - sigmaArray_mode_y[j] = new TH1D**[TotalNSamples](); + if (DoByMode) { + sigmaArray_mode_x[j] = new TH1D **[TotalNSamples](); + sigmaArray_mode_y[j] = new TH1D **[TotalNSamples](); } SampleIterator = 0; // Get each sample and how it's responded to our reweighted parameter - for(unsigned int ivs = 0; ivs < samples.size(); ivs++ ) - { - for (int k = 0; k < samples[ivs]->GetNsamples(); ++k) - { + for (unsigned int ivs = 0; ivs < samples.size(); ivs++) { + for (int k = 0; k < samples[ivs]->GetNsamples(); ++k) { // Make a string of the double std::ostringstream ss; ss << paramVal; std::string parVarTitle = name + "_" + ss.str(); - auto currSamp = M3::Clone(static_cast(samples[ivs]->GetPDF(k))); + auto currSamp = M3::Clone( + static_cast(samples[ivs]->GetPDF(k))); // Set a descriptiv-ish title - std::string title_long = std::string(currSamp->GetName())+"_"+parVarTitle; + std::string title_long = + std::string(currSamp->GetName()) + "_" + parVarTitle; // Enable the mode histograms AFTER addSelection is called - //Get the 1d binning we want. Let's just use SetupBinning to get this as it already exists + // Get the 1d binning we want. Let's just use SetupBinning to get + // this as it already exists std::vector xbins; std::vector ybins; samples[ivs]->SetupBinning(M3::int_t(k), xbins, ybins); - //KS:here we loop over all reaction modes defined in "RelevantModes[nRelevantModes]" - if (DoByMode) - { - //KS: this is only relevant if PlotByMode is turned on - //Checking each mode is time consuming so we only consider one which are relevant for particular analysis - MaCh3Modes_t RelevantModes[nRelevantModes] = {samples[ivs]->GetMaCh3Modes()->GetMode("CCQE"), samples[ivs]->GetMaCh3Modes()->GetMode("2p2h")}; - - sigmaArray_mode_x[j][SampleIterator] = new TH1D*[nRelevantModes](); - sigmaArray_mode_y[j][SampleIterator] = new TH1D*[nRelevantModes](); + // KS:here we loop over all reaction modes defined in + // "RelevantModes[nRelevantModes]" + if (DoByMode) { + // KS: this is only relevant if PlotByMode is turned on + // Checking each mode is time consuming so we only consider one + // which are relevant for particular analysis + MaCh3Modes_t RelevantModes[nRelevantModes] = { + samples[ivs]->GetMaCh3Modes()->GetMode("CCQE"), + samples[ivs]->GetMaCh3Modes()->GetMode("2p2h")}; + + sigmaArray_mode_x[j][SampleIterator] = + new TH1D *[nRelevantModes](); + sigmaArray_mode_y[j][SampleIterator] = + new TH1D *[nRelevantModes](); // Now get the TH2D mode variations std::string mode_title_long; - for(int ir = 0; ir < nRelevantModes; ir++) - { - auto currSampMode = M3::Clone(static_cast(samples[ivs]->GetPDFMode(k, RelevantModes[ir]))); + for (int ir = 0; ir < nRelevantModes; ir++) { + auto currSampMode = M3::Clone(static_cast( + samples[ivs]->GetPDFMode(k, RelevantModes[ir]))); - mode_title_long = title_long + "_" + samples[ivs]->GetMaCh3Modes()->GetMaCh3ModeName(RelevantModes[ir]); - currSampMode->SetNameTitle(mode_title_long.c_str(), mode_title_long.c_str()); + mode_title_long = + title_long + "_" + + samples[ivs]->GetMaCh3Modes()->GetMaCh3ModeName( + RelevantModes[ir]); + currSampMode->SetNameTitle(mode_title_long.c_str(), + mode_title_long.c_str()); dirArrySample[SampleIterator]->cd(); currSampMode->Write(); - sigmaArray_mode_x[j][SampleIterator][ir] = PolyProjectionX(currSampMode.get(), (mode_title_long+"_xProj").c_str(), xbins); + sigmaArray_mode_x[j][SampleIterator][ir] = PolyProjectionX( + currSampMode.get(), (mode_title_long + "_xProj").c_str(), + xbins); sigmaArray_mode_x[j][SampleIterator][ir]->SetDirectory(nullptr); - sigmaArray_mode_y[j][SampleIterator][ir] = PolyProjectionY(currSampMode.get(), (mode_title_long+"_yProj").c_str(), ybins); + sigmaArray_mode_y[j][SampleIterator][ir] = PolyProjectionY( + currSampMode.get(), (mode_title_long + "_yProj").c_str(), + ybins); sigmaArray_mode_y[j][SampleIterator][ir]->SetDirectory(nullptr); } } - //KS: This will give different results depending if data or Asimov, both have their uses - if (PlotLLHperBin) - { - auto currLLHSamp = M3::Clone(static_cast(samples[ivs]->GetPDF(k))); + // KS: This will give different results depending if data or Asimov, + // both have their uses + if (PlotLLHperBin) { + auto currLLHSamp = M3::Clone( + static_cast(samples[ivs]->GetPDF(k))); currLLHSamp->Reset(""); currLLHSamp->Fill(0.0, 0.0, 0.0); - TH2Poly* MCpdf = static_cast(samples[ivs]->GetPDF(k)); - TH2Poly* Datapdf = static_cast(samples[ivs]->GetData(k)); - TH2Poly* W2pdf = samples[ivs]->GetW2(k); + TH2Poly *MCpdf = static_cast(samples[ivs]->GetPDF(k)); + TH2Poly *Datapdf = + static_cast(samples[ivs]->GetData(k)); + TH2Poly *W2pdf = samples[ivs]->GetW2(k); - for(int bin = 1; bin < currLLHSamp->GetNumberOfBins()+1; bin++) - { + for (int bin = 1; bin < currLLHSamp->GetNumberOfBins() + 1; + bin++) { const double mc = MCpdf->GetBinContent(bin); const double dat = Datapdf->GetBinContent(bin); const double w2 = W2pdf->GetBinContent(bin); - currLLHSamp->SetBinContent(bin, samples[ivs]->GetTestStatLLH(dat, mc, w2)); + currLLHSamp->SetBinContent( + bin, samples[ivs]->GetTestStatLLH(dat, mc, w2)); } - currLLHSamp->SetNameTitle((title_long+"_LLH").c_str() ,(title_long+"_LLH").c_str()); + currLLHSamp->SetNameTitle((title_long + "_LLH").c_str(), + (title_long + "_LLH").c_str()); dirArrySample[SampleIterator]->cd(); currLLHSamp->Write(); } // Project down onto x axis - sigmaArray_x[j][SampleIterator] = std::unique_ptr(PolyProjectionX(currSamp.get(), (title_long+"_xProj").c_str(), xbins)); + sigmaArray_x[j][SampleIterator] = + std::unique_ptr(PolyProjectionX( + currSamp.get(), (title_long + "_xProj").c_str(), xbins)); sigmaArray_x[j][SampleIterator]->SetDirectory(nullptr); - sigmaArray_x[j][SampleIterator]->GetXaxis()->SetTitle(currSamp->GetXaxis()->GetTitle()); - sigmaArray_y[j][SampleIterator] = std::unique_ptr(PolyProjectionY(currSamp.get(), (title_long+"_yProj").c_str(), ybins)); + sigmaArray_x[j][SampleIterator]->GetXaxis()->SetTitle( + currSamp->GetXaxis()->GetTitle()); + sigmaArray_y[j][SampleIterator] = + std::unique_ptr(PolyProjectionY( + currSamp.get(), (title_long + "_yProj").c_str(), ybins)); sigmaArray_y[j][SampleIterator]->SetDirectory(nullptr); - sigmaArray_y[j][SampleIterator]->GetXaxis()->SetTitle(currSamp->GetYaxis()->GetTitle()); + sigmaArray_y[j][SampleIterator]->GetXaxis()->SetTitle( + currSamp->GetYaxis()->GetTitle()); - sigmaArray_x_norm[j][SampleIterator] = M3::Clone(sigmaArray_x[j][SampleIterator].get()); + sigmaArray_x_norm[j][SampleIterator] = + M3::Clone(sigmaArray_x[j][SampleIterator].get()); sigmaArray_x_norm[j][SampleIterator]->Scale(1., "width"); - sigmaArray_y_norm[j][SampleIterator] = M3::Clone(sigmaArray_y[j][SampleIterator].get()); + sigmaArray_y_norm[j][SampleIterator] = + M3::Clone(sigmaArray_y[j][SampleIterator].get()); sigmaArray_y_norm[j][SampleIterator]->SetDirectory(nullptr); sigmaArray_y_norm[j][SampleIterator]->Scale(1., "width"); @@ -1272,7 +1319,7 @@ void FitterBase::RunSigmaVar() { sigmaArray_x[j][k]->Write(); sigmaArray_y[j][k]->Write(); SampleIterator++; - }//End loop over samples + } // End loop over samples } } // End looping over variation @@ -1281,16 +1328,29 @@ void FitterBase::RunSigmaVar() { SampleIterator = 0; // Get each sample and how it's responded to our reweighted parameter - for(unsigned int ivs = 0; ivs < samples.size(); ivs++ ) - { - for (int k = 0; k < samples[ivs]->GetNsamples(); ++k) - { - std::string title = std::string(samples[ivs]->GetPDF(k)->GetName()) + "_" + name; - auto var_x = MakeAsymGraph(sigmaArray_x[1][SampleIterator].get(), sigmaArray_x[2][SampleIterator].get(), sigmaArray_x[3][SampleIterator].get(), (title+"_X").c_str()); - auto var_y = MakeAsymGraph(sigmaArray_y[1][SampleIterator].get(), sigmaArray_y[2][SampleIterator].get(), sigmaArray_y[3][SampleIterator].get(), (title+"_Y").c_str()); - - auto var_x_norm = MakeAsymGraph(sigmaArray_x_norm[1][SampleIterator].get(), sigmaArray_x_norm[2][SampleIterator].get(), sigmaArray_x_norm[3][SampleIterator].get(), (title+"_X_norm").c_str()); - auto var_y_norm = MakeAsymGraph(sigmaArray_y_norm[1][SampleIterator].get(), sigmaArray_y_norm[2][SampleIterator].get(), sigmaArray_y_norm[3][SampleIterator].get(), (title+"_Y_norm").c_str()); + for (unsigned int ivs = 0; ivs < samples.size(); ivs++) { + for (int k = 0; k < samples[ivs]->GetNsamples(); ++k) { + std::string title = + std::string(samples[ivs]->GetPDF(k)->GetName()) + "_" + name; + auto var_x = MakeAsymGraph(sigmaArray_x[1][SampleIterator].get(), + sigmaArray_x[2][SampleIterator].get(), + sigmaArray_x[3][SampleIterator].get(), + (title + "_X").c_str()); + auto var_y = MakeAsymGraph(sigmaArray_y[1][SampleIterator].get(), + sigmaArray_y[2][SampleIterator].get(), + sigmaArray_y[3][SampleIterator].get(), + (title + "_Y").c_str()); + + auto var_x_norm = + MakeAsymGraph(sigmaArray_x_norm[1][SampleIterator].get(), + sigmaArray_x_norm[2][SampleIterator].get(), + sigmaArray_x_norm[3][SampleIterator].get(), + (title + "_X_norm").c_str()); + auto var_y_norm = + MakeAsymGraph(sigmaArray_y_norm[1][SampleIterator].get(), + sigmaArray_y_norm[2][SampleIterator].get(), + sigmaArray_y_norm[3][SampleIterator].get(), + (title + "_Y_norm").c_str()); dirArrySample[SampleIterator]->cd(); var_x->Write(); @@ -1298,17 +1358,35 @@ void FitterBase::RunSigmaVar() { var_x_norm->Write(); var_y_norm->Write(); - //KS: here we loop over all reaction modes defined in "RelevantModes[nRelevantModes]" - if (DoByMode) - { - //KS: this is only relevant if PlotByMode is turned on - //Checking each mode is time consuming so we only consider one which are relevant for particular analysis - MaCh3Modes_t RelevantModes[nRelevantModes] = {samples[ivs]->GetMaCh3Modes()->GetMode("CCQE"), samples[ivs]->GetMaCh3Modes()->GetMode("2p2h")}; - - for(int ir = 0; ir < nRelevantModes;ir++) - { - auto var_mode_x = MakeAsymGraph(sigmaArray_mode_x[1][SampleIterator][ir], sigmaArray_mode_x[2][SampleIterator][ir], sigmaArray_mode_x[3][SampleIterator][ir], (title+"_"+samples[ivs]->GetMaCh3Modes()->GetMaCh3ModeName(RelevantModes[ir])+"_X").c_str()); - auto var_mode_y = MakeAsymGraph(sigmaArray_mode_y[1][SampleIterator][ir], sigmaArray_mode_y[2][SampleIterator][ir], sigmaArray_mode_y[3][SampleIterator][ir], (title+"_"+samples[ivs]->GetMaCh3Modes()->GetMaCh3ModeName(RelevantModes[ir])+"_Y").c_str()); + // KS: here we loop over all reaction modes defined in + // "RelevantModes[nRelevantModes]" + if (DoByMode) { + // KS: this is only relevant if PlotByMode is turned on + // Checking each mode is time consuming so we only consider one + // which are relevant for particular analysis + MaCh3Modes_t RelevantModes[nRelevantModes] = { + samples[ivs]->GetMaCh3Modes()->GetMode("CCQE"), + samples[ivs]->GetMaCh3Modes()->GetMode("2p2h")}; + + for (int ir = 0; ir < nRelevantModes; ir++) { + auto var_mode_x = MakeAsymGraph( + sigmaArray_mode_x[1][SampleIterator][ir], + sigmaArray_mode_x[2][SampleIterator][ir], + sigmaArray_mode_x[3][SampleIterator][ir], + (title + "_" + + samples[ivs]->GetMaCh3Modes()->GetMaCh3ModeName( + RelevantModes[ir]) + + "_X") + .c_str()); + auto var_mode_y = MakeAsymGraph( + sigmaArray_mode_y[1][SampleIterator][ir], + sigmaArray_mode_y[2][SampleIterator][ir], + sigmaArray_mode_y[3][SampleIterator][ir], + (title + "_" + + samples[ivs]->GetMaCh3Modes()->GetMaCh3ModeName( + RelevantModes[ir]) + + "_Y") + .c_str()); dirArrySample[SampleIterator]->cd(); var_mode_x->Write(); @@ -1317,13 +1395,11 @@ void FitterBase::RunSigmaVar() { } // end if mode SampleIterator++; - }//End loop over samples(k) - }//end looping over sample object + } // End loop over samples(k) + } // end looping over sample object SampleIterator = 0; - for(unsigned int ivs = 0; ivs < samples.size(); ivs++ ) - { - for (int k = 0; k < samples[ivs]->GetNsamples(); ++k) - { + for (unsigned int ivs = 0; ivs < samples.size(); ivs++) { + for (int k = 0; k < samples[ivs]->GetNsamples(); ++k) { dirArrySample[SampleIterator]->Close(); delete dirArrySample[SampleIterator]; SampleIterator++; @@ -1333,24 +1409,19 @@ void FitterBase::RunSigmaVar() { dirArryDial->Close(); delete dirArryDial; - if (DoByMode) - { - for (int j = 0; j < numVar; ++j) - { + if (DoByMode) { + for (int j = 0; j < numVar; ++j) { SampleIterator = 0; - for(unsigned int ivs = 0; ivs < samples.size(); ivs++ ) - { - for (int k = 0; k < samples[ivs]->GetNsamples(); ++k) - { - for(int ir = 0; ir < nRelevantModes;ir++) - { + for (unsigned int ivs = 0; ivs < samples.size(); ivs++) { + for (int k = 0; k < samples[ivs]->GetNsamples(); ++k) { + for (int ir = 0; ir < nRelevantModes; ir++) { delete sigmaArray_mode_x[j][SampleIterator][ir]; delete sigmaArray_mode_y[j][SampleIterator][ir]; - }// end for nRelevantModes + } // end for nRelevantModes delete[] sigmaArray_mode_x[j][SampleIterator]; delete[] sigmaArray_mode_y[j][SampleIterator]; SampleIterator++; - }// end for samples + } // end for samples } } delete[] sigmaArray_mode_x; @@ -1360,12 +1431,14 @@ void FitterBase::RunSigmaVar() { } // end looping over covarianceBase objects } - -// ************************* -// For comparison with P-Theta we usually have to apply different parameter values then usual 1, 3 sigma -void FitterBase::CustomRange(const std::string& ParName, const double sigma, double& ParamShiftValue) const { // ************************* - if(!fitMan->raw()["SigmaVar"]["CustomRange"]) return; +// For comparison with P-Theta we usually have to apply different parameter +// values then usual 1, 3 sigma +void FitterBase::CustomRange(const std::string &ParName, const double sigma, + double &ParamShiftValue) const { + // ************************* + if (!fitMan->raw()["SigmaVar"]["CustomRange"]) + return; auto Config = fitMan->raw()["SigmaVar"]["CustomRange"]; @@ -1373,15 +1446,17 @@ void FitterBase::CustomRange(const std::string& ParName, const double sigma, dou if (Config[ParName] && Config[ParName][sigmaStr]) { ParamShiftValue = Config[ParName][sigmaStr].as(); - MACH3LOG_INFO(" ::: setting custom range from config ::: {} -> {}", ParName, ParamShiftValue); + MACH3LOG_INFO(" ::: setting custom range from config ::: {} -> {}", + ParName, ParamShiftValue); } } // ************************* /// Helper to write histograms -void WriteHistograms(TH1 *hist, const std::string& baseName) { -// ************************* - if (!hist) return; +void WriteHistograms(TH1 *hist, const std::string &baseName) { + // ************************* + if (!hist) + return; hist->SetTitle(baseName.c_str()); // Get the class name of the histogram TString className = hist->ClassName(); @@ -1398,50 +1473,60 @@ void WriteHistograms(TH1 *hist, const std::string& baseName) { // ************************* /// Generic histogram writer - should make main code more palatable -void WriteHistogramsByMode(SampleHandlerFD *sample, - const std::string& suffix, - const bool by_mode, - const bool by_channel, - const std::vector& SampleDir) { -// ************************* +void WriteHistogramsByMode(SampleHandlerFD *sample, const std::string &suffix, + const bool by_mode, const bool by_channel, + const std::vector &SampleDir) { + // ************************* MaCh3Modes *modes = sample->GetMaCh3Modes(); for (int iSample = 0; iSample < sample->GetNsamples(); ++iSample) { SampleDir[iSample]->cd(); const std::string sampleName = sample->GetSampleTitle(iSample); - for(int iDim = 0; iDim < sample->GetNDim(iSample); iDim++) { + for (int iDim = 0; iDim < sample->GetNDim(iSample); iDim++) { std::string ProjectionName = ""; std::string ProjectionSuffix = "_1DProj" + std::to_string(iDim); - if(iDim == 0) { + if (iDim == 0) { ProjectionName = sample->GetXBinVarName(iSample); } else if (iDim == 1) { ProjectionName = sample->GetYBinVarName(iSample); } else { - MACH3LOG_ERROR("Not yet implemented for dimension {}", iDim+1); + MACH3LOG_ERROR("Not yet implemented for dimension {}", iDim + 1); throw MaCh3Exception(__FILE__, __LINE__); } // Probably a better way of handling this logic if (by_mode) { for (int iMode = 0; iMode < modes->GetNModes(); ++iMode) { - auto modeHist = sample->Get1DVarHistByModeAndChannel(iSample, ProjectionName, iMode); - WriteHistograms(modeHist, sampleName + "_" + modes->GetMaCh3ModeName(iMode) + ProjectionSuffix + suffix); + auto modeHist = sample->Get1DVarHistByModeAndChannel( + iSample, ProjectionName, iMode); + WriteHistograms(modeHist, sampleName + "_" + + modes->GetMaCh3ModeName(iMode) + + ProjectionSuffix + suffix); delete modeHist; } } if (by_channel) { for (int iChan = 0; iChan < sample->GetNOscChannels(iSample); ++iChan) { - auto chanHist = sample->Get1DVarHistByModeAndChannel(iSample, ProjectionName, -1, iChan); // -1 skips over mode plotting - WriteHistograms(chanHist, sampleName + "_" + sample->GetFlavourName(iSample, iChan) + ProjectionSuffix + suffix); + auto chanHist = sample->Get1DVarHistByModeAndChannel( + iSample, ProjectionName, -1, + iChan); // -1 skips over mode plotting + WriteHistograms(chanHist, sampleName + "_" + + sample->GetFlavourName(iSample, iChan) + + ProjectionSuffix + suffix); delete chanHist; } } if (by_mode && by_channel) { for (int iMode = 0; iMode < modes->GetNModes(); ++iMode) { - for (int iChan = 0; iChan < sample->GetNOscChannels(iSample); ++iChan) { - auto hist = sample->Get1DVarHistByModeAndChannel(iSample, ProjectionName, iMode, iChan); - WriteHistograms(hist, sampleName + "_" + modes->GetMaCh3ModeName(iMode) + "_" + sample->GetFlavourName(iSample, iChan) + ProjectionSuffix + suffix); + for (int iChan = 0; iChan < sample->GetNOscChannels(iSample); + ++iChan) { + auto hist = sample->Get1DVarHistByModeAndChannel( + iSample, ProjectionName, iMode, iChan); + WriteHistograms(hist, sampleName + "_" + + modes->GetMaCh3ModeName(iMode) + "_" + + sample->GetFlavourName(iSample, iChan) + + ProjectionSuffix + suffix); delete hist; } } @@ -1452,8 +1537,10 @@ void WriteHistogramsByMode(SampleHandlerFD *sample, WriteHistograms(hist, sampleName + ProjectionSuffix + suffix); delete hist; // Only for 2D - if(iDim == 1) { - auto hist2D = sample->Get2DVarHist(iSample, sample->GetXBinVarName(iSample), sample->GetYBinVarName(iSample)); + if (iDim == 1) { + auto hist2D = + sample->Get2DVarHist(iSample, sample->GetXBinVarName(iSample), + sample->GetYBinVarName(iSample)); WriteHistograms(hist2D, sampleName + "_2DProj" + suffix); delete hist2D; } @@ -1464,70 +1551,83 @@ void WriteHistogramsByMode(SampleHandlerFD *sample, // ************************* void FitterBase::RunSigmaVarFD() { -// ************************* + // ************************* // Save the settings into the output file SaveSettings(); - bool plot_by_mode = GetFromManager(fitMan->raw()["SigmaVar"]["PlotByMode"], false); - bool plot_by_channel = GetFromManager(fitMan->raw()["SigmaVar"]["PlotByChannel"], false); - auto SkipVector = GetFromManager>(fitMan->raw()["SigmaVar"]["SkipVector"], {}, __FILE__ , __LINE__); - - if (plot_by_mode) MACH3LOG_INFO("Plotting by sample and mode"); - if (plot_by_channel) MACH3LOG_INFO("Plotting by sample and channel"); - if (!plot_by_mode && !plot_by_channel) MACH3LOG_INFO("Plotting by sample only"); - if (plot_by_mode && plot_by_channel) MACH3LOG_INFO("Plotting by sample, mode and channel"); - - auto SigmaArray = GetFromManager>(fitMan->raw()["SigmaVar"]["SigmaArray"], {-3, -1, 0, 1, 3}, __FILE__ , __LINE__); - if (std::find(SigmaArray.begin(), SigmaArray.end(), 0.0) == SigmaArray.end()) { - MACH3LOG_ERROR(":: SigmaArray does not contain 0! Current contents: {} ::", fmt::join(SigmaArray, ", ")); + bool plot_by_mode = + GetFromManager(fitMan->raw()["SigmaVar"]["PlotByMode"], false); + bool plot_by_channel = + GetFromManager(fitMan->raw()["SigmaVar"]["PlotByChannel"], false); + auto SkipVector = GetFromManager>( + fitMan->raw()["SigmaVar"]["SkipVector"], {}, __FILE__, __LINE__); + + if (plot_by_mode) + MACH3LOG_INFO("Plotting by sample and mode"); + if (plot_by_channel) + MACH3LOG_INFO("Plotting by sample and channel"); + if (!plot_by_mode && !plot_by_channel) + MACH3LOG_INFO("Plotting by sample only"); + if (plot_by_mode && plot_by_channel) + MACH3LOG_INFO("Plotting by sample, mode and channel"); + + auto SigmaArray = GetFromManager>( + fitMan->raw()["SigmaVar"]["SigmaArray"], {-3, -1, 0, 1, 3}, __FILE__, + __LINE__); + if (std::find(SigmaArray.begin(), SigmaArray.end(), 0.0) == + SigmaArray.end()) { + MACH3LOG_ERROR(":: SigmaArray does not contain 0! Current contents: {} ::", + fmt::join(SigmaArray, ", ")); throw MaCh3Exception(__FILE__, __LINE__); } - TDirectory* SigmaDir = outputFile->mkdir("SigmaVar"); + TDirectory *SigmaDir = outputFile->mkdir("SigmaVar"); outputFile->cd(); - for (size_t s = 0; s < systematics.size(); ++s) - { - for(int i = 0; i < systematics[s]->GetNumParams(); i++) - { + for (size_t s = 0; s < systematics.size(); ++s) { + for (int i = 0; i < systematics[s]->GetNumSystematicParams(); i++) { std::string ParName = systematics[s]->GetParFancyName(i); // KS: Check if we want to skip this parameter - if(CheckSkipParameter(SkipVector, ParName)) continue; + if (CheckSkipParameter(SkipVector, ParName)) + continue; MACH3LOG_INFO(":: Param {} ::", systematics[s]->GetParFancyName(i)); - TDirectory* ParamDir = SigmaDir->mkdir(ParName.c_str()); + TDirectory *ParamDir = SigmaDir->mkdir(ParName.c_str()); ParamDir->cd(); const double ParamNomValue = systematics[s]->GetParProp(i); const double ParamLower = systematics[s]->GetLowerBound(i); const double ParamUpper = systematics[s]->GetUpperBound(i); - for(unsigned int iSample = 0; iSample < samples.size(); ++iSample) - { - auto* MaCh3Sample = dynamic_cast(samples[iSample]); + for (unsigned int iSample = 0; iSample < samples.size(); ++iSample) { + auto *MaCh3Sample = dynamic_cast(samples[iSample]); if (!MaCh3Sample) { - MACH3LOG_ERROR(":: Sample {} do not inherit from SampleHandlerFD this is not implemented::", samples[i]->GetSampleTitle(iSample)); + MACH3LOG_ERROR(":: Sample {} do not inherit from SampleHandlerFD " + "this is not implemented::", + samples[i]->GetSampleTitle(iSample)); throw MaCh3Exception(__FILE__, __LINE__); } - std::vector SampleDir(MaCh3Sample->GetNsamples()); - for (int SampleIndex = 0; SampleIndex < MaCh3Sample->GetNsamples(); ++SampleIndex) { - SampleDir[SampleIndex] = ParamDir->mkdir(MaCh3Sample->GetSampleTitle(SampleIndex).c_str()); + std::vector SampleDir(MaCh3Sample->GetNsamples()); + for (int SampleIndex = 0; SampleIndex < MaCh3Sample->GetNsamples(); + ++SampleIndex) { + SampleDir[SampleIndex] = + ParamDir->mkdir(MaCh3Sample->GetSampleTitle(SampleIndex).c_str()); } for (size_t j = 0; j < SigmaArray.size(); ++j) { double sigma = SigmaArray[j]; - double ParamShiftValue = ParamNomValue + sigma * std::sqrt((*systematics[s]->GetCovMatrix())(i,i)); - ParamShiftValue = std::max(std::min(ParamShiftValue, ParamUpper), ParamLower); + double ParamShiftValue = + ParamNomValue + + sigma * std::sqrt((*systematics[s]->GetCovMatrix())(i, i)); + ParamShiftValue = + std::max(std::min(ParamShiftValue, ParamUpper), ParamLower); /// Apply custom range to make easier comparison with p-theta CustomRange(ParName, sigma, ParamShiftValue); - MACH3LOG_INFO( - " - set to {:<5.2f} ({:<2} sigma shift)", - ParamShiftValue, - sigma - ); + MACH3LOG_INFO(" - set to {:<5.2f} ({:<2} sigma shift)", + ParamShiftValue, sigma); systematics[s]->SetParProp(i, ParamShiftValue); @@ -1544,15 +1644,18 @@ void FitterBase::RunSigmaVarFD() { suffix = "_" + ParName + "_nom_val_" + valueStr; } else { std::string sign = (sigma > 0) ? "p" : "n"; - suffix = "_" + ParName + "_sig_" + sign + sigmaStr + "_val_" + valueStr; + suffix = + "_" + ParName + "_sig_" + sign + sigmaStr + "_val_" + valueStr; } systematics[s]->SetParProp(i, ParamShiftValue); MaCh3Sample->Reweight(); - WriteHistogramsByMode(MaCh3Sample, suffix, plot_by_mode, plot_by_channel, SampleDir); + WriteHistogramsByMode(MaCh3Sample, suffix, plot_by_mode, + plot_by_channel, SampleDir); } - for (int subSampleIndex = 0; subSampleIndex < MaCh3Sample->GetNsamples(); ++subSampleIndex) { + for (int subSampleIndex = 0; + subSampleIndex < MaCh3Sample->GetNsamples(); ++subSampleIndex) { SampleDir[subSampleIndex]->Close(); delete SampleDir[subSampleIndex]; } diff --git a/Fitters/LikelihoodFit.cpp b/Fitters/LikelihoodFit.cpp index 66b24e91e..b0a64feac 100644 --- a/Fitters/LikelihoodFit.cpp +++ b/Fitters/LikelihoodFit.cpp @@ -3,42 +3,66 @@ // ******************* // Run the Markov chain with all the systematic objects added LikelihoodFit::LikelihoodFit(manager *man) : FitterBase(man) { -// ******************* - NPars = 0; - NParsPCA = 0; - fMirroring = GetFromManager(fitMan->raw()["General"]["Fitter"]["Mirroring"], false); - if(fMirroring) MACH3LOG_INFO("Mirroring enabled"); + // ******************* + fMirroring = GetFromManager( + fitMan->raw()["General"]["Fitter"]["Mirroring"], false); + if (fMirroring) + MACH3LOG_INFO("Mirroring enabled"); } - // ************************* // Destructor: close the logger and output file LikelihoodFit::~LikelihoodFit() { -// ************************* - + // ************************* } // ******************* void LikelihoodFit::PrepareFit() { -// ******************* + // ******************* // Save the settings into the output file SaveSettings(); // Prepare the output branches PrepareOutput(); - - for (size_t s = 0; s < systematics.size(); ++s) { - NPars += systematics[s]->GetNumParams(); - NParsPCA += systematics[s]->GetNParameters(); + + for (auto phandlr : systematics) { + NPars += phandlr->GetNumSystematicParams(); + NParsPCA += phandlr->GetNumProposalParams(); } - //KS: If PCA is note enabled NParsPCA == NPars - MACH3LOG_INFO("Total number of parameters {}", NParsPCA); + MACH3LOG_INFO("Total number of parameters {}", NPars); } -// ******************* -double LikelihoodFit::CalcChi2(const double* x) { -// ******************* +// parameters come in in the sampler basis +// ******************* + +double LikelihoodFit::CalcChi2PC(const double *x) { + + int ParCounter = 0; + static std::vector syst_par_values(NPars, 0); + + auto par_it = syst_par_values.begin(); + for (auto &parhandlr : systematics) { + + // set the parameters in the sampler basis + std::copy_n(x + ParCounter, parhandlr->GetNumProposalParams(), + parhandlr->proposer.params.proposed.data()); + ParCounter += parhandlr->GetNumProposalParams(); + + // this accepts the step for the proposer and rotates the parameters back to + // the systematic basis + parhandlr->AcceptStep(); + + std::copy_n(parhandlr->GetParCurrVec().begin(), + parhandlr->GetNumSystematicParams(), par_it); + std::advance(par_it, parhandlr->GetNumSystematicParams()); + } + + return CalcChi2(syst_par_values.data()); +} + +double LikelihoodFit::CalcChi2(const double *x) { + // ******************* if (step % 10000 == 0) { MACH3LOG_INFO("Iteration {}", step); } @@ -46,77 +70,57 @@ double LikelihoodFit::CalcChi2(const double* x) { stepClock->Start(); int ParCounter = 0; - double llh = 0; - for (std::vector::iterator it = systematics.begin(); it != systematics.end(); ++it) - { - if(!(*it)->IsPCA()) - { - std::vector pars; - const int NumPar = (*it)->GetNumParams(); - //KS: Avoid push back as they are slow - pars.resize(NumPar); - for(int i = 0; i < NumPar; ++i, ++ParCounter) - { - double ParVal = x[ParCounter]; - //KS: Basically apply mirroring for parameters out of bounds - if(fMirroring) - { - if(ParVal < (*it)->GetLowerBound(i)) - { - ParVal = (*it)->GetLowerBound(i) + ((*it)->GetLowerBound(i) - ParVal); - } - else if (ParVal > (*it)->GetUpperBound(i)) - { - ParVal = (*it)->GetUpperBound(i) - ( ParVal - (*it)->GetUpperBound(i)); - } + for (auto &parhandlr : systematics) { + for (int i = 0; i < parhandlr->GetNumSystematicParams(); ++i) { + double ParVal = x[ParCounter++]; + // KS: Basically apply mirroring for parameters out of bounds + if (fMirroring) { // mirror in the systematic basis as it is where the + // bounds are defined + if (ParVal < parhandlr->GetLowerBound(i)) { + ParVal = parhandlr->GetLowerBound(i) + + (parhandlr->GetLowerBound(i) - ParVal); + } else if (ParVal > parhandlr->GetUpperBound(i)) { + ParVal = parhandlr->GetUpperBound(i) - + (ParVal - parhandlr->GetUpperBound(i)); } - pars[i] = ParVal; } - (*it)->SetParameters(pars); + parhandlr->SetParCurrProp(i, ParVal); } - else - { - std::vector pars; - const int NumPar = (*it)->GetNParameters(); - //KS: Avoid push back as they are slow - pars.resize(NumPar); - for(int i = 0; i < NumPar; ++i, ++ParCounter) - { - double ParVal = x[ParCounter]; - //KS: Basically apply mirroring for parameters out of bounds - pars[i] = ParVal; - } - (*it)->GetPCAHandler()->SetParametersPCA(pars); - } - (*it)->AcceptStep(); } + double llh = 0; + // Loop over the systematics and propose the initial step int stdIt = 0; - for (std::vector::iterator it = systematics.begin(); it != systematics.end(); ++it, ++stdIt) - { - //GetLikelihood will return LargeL if out of bounds, for minimizers this is not the problem, while calcLikelihood will return actual likelihood - syst_llh[stdIt] = (*it)->CalcLikelihood(); + for (auto &parhandlr : systematics) { + + // GetLikelihood will return LargeL if out of bounds, for minimizers this is + // not the problem, while calcLikelihood will return actual likelihood + syst_llh[stdIt] = parhandlr->CalcLikelihood(); llh += syst_llh[stdIt]; - #ifdef DEBUG - if (debug) debugFile << "LLH after " << systematics[stdIt]->GetName() << " " << llh << std::endl; - #endif +#ifdef DEBUG + if (debug) + debugFile << "LLH after " << systematics[stdIt]->GetName() << " " << llh + << std::endl; +#endif + stdIt++; } // Could multi-thread this // But since sample reweight is multi-threaded it's probably better to do that - for (size_t i = 0; i < samples.size(); i++) - { + for (size_t i = 0; i < samples.size(); i++) { samples[i]->Reweight(); } - //DB for atmospheric event by event sample migration, need to fully reweight all samples to allow event passing prior to likelihood evaluation + // DB for atmospheric event by event sample migration, need to fully reweight + // all samples to allow event passing prior to likelihood evaluation for (size_t i = 0; i < samples.size(); i++) { // Get the sample likelihoods and add them sample_llh[i] = samples[i]->GetLikelihood(); llh += sample_llh[i]; - #ifdef DEBUG - if (debug) debugFile << "LLH after sample " << i << " " << llh << std::endl; - #endif +#ifdef DEBUG + if (debug) + debugFile << "LLH after sample " << i << " " << llh << std::endl; +#endif } // Save the proposed likelihood (class member) @@ -131,11 +135,11 @@ double LikelihoodFit::CalcChi2(const double* x) { outTree->Fill(); // Auto save the output - if (step % auto_save == 0) outTree->AutoSave(); + if (step % auto_save == 0) + outTree->AutoSave(); step++; accCount++; - llh = 2.0*llh; + llh = 2.0 * llh; return llh; } - diff --git a/Fitters/LikelihoodFit.h b/Fitters/LikelihoodFit.h index 1fd8baaa7..d8530bf44 100644 --- a/Fitters/LikelihoodFit.h +++ b/Fitters/LikelihoodFit.h @@ -12,6 +12,8 @@ class LikelihoodFit : public FitterBase { /// @brief Chi2 calculation over all included samples and syst objects virtual double CalcChi2(const double* x); + //Expects parameters to be in proposal/PCA basis, transforms back before calling CalcChi2 + virtual double CalcChi2PC(const double* x); /// @brief Get total number of params, this sums over all covariance objects inline int GetNPars(){return NPars;}; diff --git a/Fitters/MCMCBase.cpp b/Fitters/MCMCBase.cpp index 1d04870a8..c88b74a20 100644 --- a/Fitters/MCMCBase.cpp +++ b/Fitters/MCMCBase.cpp @@ -4,202 +4,201 @@ // Initialise the manager and make it an object of mcmc class // Now we can dump manager settings to the output file MCMCBase::MCMCBase(manager *man) : FitterBase(man) { -// ************************* - // Beginning step number - stepStart = 0; - - // Starting parameters should be thrown - out_of_bounds = false; - chainLength = Get(fitMan->raw()["General"]["MCMC"]["NSteps"], __FILE__, __LINE__); - - AnnealTemp = GetFromManager(fitMan->raw()["General"]["MCMC"]["AnnealTemp"], -999); - if (AnnealTemp < 0) - anneal = false; - else - { - MACH3LOG_INFO("Enabling simulated annealing with T = {}", AnnealTemp); - anneal = true; - } + // ************************* + // Beginning step number + stepStart = 0; + + // Starting parameters should be thrown + out_of_bounds = false; + chainLength = Get(fitMan->raw()["General"]["MCMC"]["NSteps"], + __FILE__, __LINE__); + + AnnealTemp = GetFromManager( + fitMan->raw()["General"]["MCMC"]["AnnealTemp"], -999); + if (AnnealTemp < 0) + anneal = false; + else { + MACH3LOG_INFO("Enabling simulated annealing with T = {}", AnnealTemp); + anneal = true; + } } - // ******************* // Run the Markov chain with all the systematic objects added void MCMCBase::RunMCMC() { -// ******************* - // Save the settings into the output file - SaveSettings(); - - // Prepare the output branches - PrepareOutput(); - - // Remove obsolete memory and make other checks before fit starts - SanitiseInputs(); - - // Print Progress before Propose Step - PrintProgress(false); - - // Reconfigure the samples, systematics and oscillation for first weight - // ProposeStep sets logLProp - ProposeStep(); - // Set the current logL to the proposed logL for the 0th step - // Accept the first step to set logLCurr: this shouldn't affect the MCMC because we ignore the first N steps in burn-in - logLCurr = logLProp; - - // Begin MCMC - const auto StepEnd = stepStart + chainLength; - for (step = stepStart; step < StepEnd; ++step) - { - DoMCMCStep(); - } - // Save all the MCMC output - SaveOutput(); - - - // Process MCMC - ProcessMCMC(); - - // Save the adaptive MCMC - for (const auto &syst : systematics) - { - if (syst->GetDoAdaption()) - { - auto adaptive_handler = syst->GetAdaptiveHandler(); - adaptive_handler->SaveAdaptiveToFile(adaptive_handler->GetOutFileName(), syst->GetName(), true); - } + // ******************* + // Save the settings into the output file + SaveSettings(); + + // Prepare the output branches + PrepareOutput(); + + // Remove obsolete memory and make other checks before fit starts + SanitiseInputs(); + + // Print Progress before Propose Step + PrintProgress(false); + + // Reconfigure the samples, systematics and oscillation for first weight + // ProposeStep sets logLProp + ProposeStep(); + // Set the current logL to the proposed logL for the 0th step + // Accept the first step to set logLCurr: this shouldn't affect the MCMC + // because we ignore the first N steps in burn-in + logLCurr = logLProp; + + // Begin MCMC + const auto StepEnd = stepStart + chainLength; + for (step = stepStart; step < StepEnd; ++step) { + DoMCMCStep(); + } + // Save all the MCMC output + SaveOutput(); + + // Process MCMC + ProcessMCMC(); + + // Save the adaptive MCMC + for (const auto &syst : systematics) { + if (syst->GetDoAdaption()) { + throw std::runtime_error("Adaption not yet implemented"); } + } } // ******************* void MCMCBase::DoMCMCStep() { -// ******************* - /// Starts step timer, prints progress - PreStepProcess(); - /// Step proposal, acceptance etc - DoStep(); - /// Tree filling etc. - PostStepProcess(); + // ******************* + /// Starts step timer, prints progress + PreStepProcess(); + /// Step proposal, acceptance etc + DoStep(); + /// Tree filling etc. + PostStepProcess(); } // ******************* void MCMCBase::PreStepProcess() { -// ******************* - stepClock->Start(); - out_of_bounds = false; - - // Print 10 steps in total - if ((step - stepStart) % (chainLength / 10) == 0) - { - PrintProgress(); - } + // ******************* + stepClock->Start(); + out_of_bounds = false; + + // Print 10 steps in total + if ((step - stepStart) % (chainLength / 10) == 0) { + PrintProgress(); + } } // ************************* void MCMCBase::PostStepProcess() { -// ************************* - stepClock->Stop(); - stepTime = stepClock->RealTime(); + // ************************* + stepClock->Stop(); + stepTime = stepClock->RealTime(); - // Write step to output tree - outTree->Fill(); + // Write step to output tree + outTree->Fill(); - // Do Adaptive MCMC - AdaptiveStep(); + // Do Adaptive MCMC + AdaptiveStep(); - if (step % auto_save == 0){ - outTree->AutoSave(); - } + if (step % auto_save == 0) { + outTree->AutoSave(); + } } // ******************* // Print the fit output progress void MCMCBase::PrintProgress(bool StepsPrint) { -// ******************* - if(StepsPrint) MACH3LOG_INFO("Step:\t{}/{}, current: {:.2f}, proposed: {:.2f}", step - stepStart, chainLength, logLCurr, logLProp); - if(StepsPrint) MACH3LOG_INFO("Accepted/Total steps: {}/{} = {:.2f}", accCount, step - stepStart, static_cast(accCount) / static_cast(step - stepStart)); - - for (size_t i = 0; i < samples.size(); ++i) { - samples[i]->PrintRates(); - } - - for (ParameterHandlerBase *cov : systematics) { - cov->PrintNominalCurrProp(); - } + // ******************* + if (StepsPrint) + MACH3LOG_INFO("Step:\t{}/{}, current: {:.2f}, proposed: {:.2f}", + step - stepStart, chainLength, logLCurr, logLProp); + if (StepsPrint) + MACH3LOG_INFO( + "Accepted/Total steps: {}/{} = {:.2f}", accCount, step - stepStart, + static_cast(accCount) / static_cast(step - stepStart)); + + for (size_t i = 0; i < samples.size(); ++i) { + samples[i]->PrintRates(); + } + + for (ParameterHandlerBase *cov : systematics) { + cov->PrintNominalCurrProp(); + } #ifdef DEBUG - if (debug) - { - debugFile << "\n-------------------------------------------------------" << std::endl; - debugFile << "Step:\t" << step + 1 << "/" << chainLength << " | current: " << logLCurr << " proposed: " << logLProp << std::endl; - } + if (debug) { + debugFile << "\n-------------------------------------------------------" + << std::endl; + debugFile << "Step:\t" << step + 1 << "/" << chainLength + << " | current: " << logLCurr << " proposed: " << logLProp + << std::endl; + } #endif } // ******************* void MCMCBase::StartFromPreviousFit(const std::string &FitName) { -// ******************* - // Use base class - FitterBase::StartFromPreviousFit(FitName); - - // For MCMC we also need to set stepStart - TFile *infile = M3::Open(FitName, "READ", __FILE__, __LINE__); - TTree *posts = infile->Get("posteriors"); - unsigned int step_val = 0; - - posts->SetBranchAddress("step", &step_val); - posts->GetEntry(posts->GetEntries() - 1); - - stepStart = step_val; - // KS: Also update number of steps if using adaption - for (unsigned int i = 0; i < systematics.size(); ++i) - { - if (systematics[i]->GetDoAdaption()) - { - systematics[i]->SetNumberOfSteps(step_val); - } + // ******************* + // Use base class + FitterBase::StartFromPreviousFit(FitName); + + // For MCMC we also need to set stepStart + TFile *infile = M3::Open(FitName, "READ", __FILE__, __LINE__); + TTree *posts = infile->Get("posteriors"); + unsigned int step_val = 0; + + posts->SetBranchAddress("step", &step_val); + posts->GetEntry(posts->GetEntries() - 1); + + stepStart = step_val; + // KS: Also update number of steps if using adaption + for (unsigned int i = 0; i < systematics.size(); ++i) { + if (systematics[i]->GetDoAdaption()) { + systematics[i]->SetNumberOfSteps(step_val); } - infile->Close(); - delete infile; + } + infile->Close(); + delete infile; } // ************************* void MCMCBase::AdaptiveStep() { -// ************************* - // Save the Adaptive output - for (const auto &syst : systematics) - { - if (syst->GetDoAdaption()){ - syst->UpdateAdaptiveCovariance(); - } + // ************************* + // Save the Adaptive output + for (const auto &syst : systematics) { + if (syst->GetDoAdaption()) { + syst->UpdateAdaptiveCovariance(); } + } } // ************************* bool MCMCBase::IsStepAccepted(const double acc_prob) { -// ************************* - // Get the random number - const double fRandom = random->Rndm(); - // Do the accept/reject - #ifdef DEBUG - debugFile << " logLProp: " << logLProp << " logLCurr: " << logLCurr << " acc_prob: " << acc_prob << " fRandom: " << fRandom << std::endl; - #endif - - if (fRandom > acc_prob) - { - // Reject - return false; - } - return true; + // ************************* + // Get the random number + const double fRandom = random->Rndm(); +// Do the accept/reject +#ifdef DEBUG + debugFile << " logLProp: " << logLProp << " logLCurr: " << logLCurr + << " acc_prob: " << acc_prob << " fRandom: " << fRandom + << std::endl; +#endif + + if (fRandom > acc_prob) { + // Reject + return false; + } + return true; } // ************************* void MCMCBase::AcceptStep() { -// ************************* - ++accCount; - logLCurr = logLProp; - - // Loop over systematics and accept - for (size_t s = 0; s < systematics.size(); ++s) - { - systematics[s]->AcceptStep(); - } + // ************************* + ++accCount; + logLCurr = logLProp; + + // Loop over systematics and accept + for (size_t s = 0; s < systematics.size(); ++s) { + systematics[s]->AcceptStep(); + } } diff --git a/Fitters/MinuitFit.cpp b/Fitters/MinuitFit.cpp index 1e54b42e1..6f46f04a9 100644 --- a/Fitters/MinuitFit.cpp +++ b/Fitters/MinuitFit.cpp @@ -3,9 +3,10 @@ // ******************* // Run the Minuit Fit with all the systematic objects added MinuitFit::MinuitFit(manager *man) : LikelihoodFit(man) { -// ******************* + // ******************* AlgorithmName = "MinuitFit"; - /// @todo KS: Make this in future configurable, for more see: https://root.cern.ch/doc/master/classROOT_1_1Math_1_1Minimizer.html + /// @todo KS: Make this in future configurable, for more see: + /// https://root.cern.ch/doc/master/classROOT_1_1Math_1_1Minimizer.html // Minimizer type: determines the underlying implementation. // Available types include: // - "Minuit2" (recommended modern option) @@ -23,71 +24,57 @@ MinuitFit::MinuitFit(manager *man) : LikelihoodFit(man) { // - "Scan" : parameter grid scan const std::string MinimizerAlgo = "Migrad"; - MACH3LOG_INFO("Creating instance of Minimizer with {} and {}", MinimizerType, MinimizerAlgo); + MACH3LOG_INFO("Creating instance of Minimizer with {} and {}", MinimizerType, + MinimizerAlgo); minuit = std::unique_ptr( - ROOT::Math::Factory::CreateMinimizer(MinimizerType.c_str(), MinimizerAlgo.c_str())); + ROOT::Math::Factory::CreateMinimizer(MinimizerType.c_str(), + MinimizerAlgo.c_str())); } // ************************* // Destructor: close the logger and output file MinuitFit::~MinuitFit() { -// ************************* + // ************************* } - // ******************* // Run the Minuit with all the systematic objects added void MinuitFit::RunMCMC() { -// ******************* + // ******************* PrepareFit(); // Remove obsolete memory and make other checks before fit starts SanitiseInputs(); - //KS: For none PCA this will be equal to normal parameters - const int NparsMinuitFull = NPars; - const int NparsMinuit = NParsPCA; - - //KS: Set SetFunction we will Minimize - ROOT::Math::Functor fChi2(this, &MinuitFit::CalcChi2, NparsMinuit); + // KS: Set SetFunction we will Minimize + ROOT::Math::Functor fChi2(this, &MinuitFit::CalcChi2PC, NParsPCA); minuit->SetFunction(fChi2); - //KS: add config or something + // KS: add config or something minuit->SetPrintLevel(2); minuit->SetTolerance(0.01); - minuit->SetMaxFunctionCalls(fitMan->raw()["General"]["Minuit2"]["NSteps"].as()); + minuit->SetMaxFunctionCalls( + fitMan->raw()["General"]["Minuit2"]["NSteps"].as()); minuit->SetMaxIterations(10000); MACH3LOG_INFO("Preparing Minuit"); int ParCounter = 0; - for (std::vector::iterator it = systematics.begin(); it != systematics.end(); ++it) - { - if(!(*it)->IsPCA()) - { - for(int i = 0; i < (*it)->GetNumParams(); ++i, ++ParCounter) - { - //KS: Index, name, prior, step scale [different to MCMC], - minuit->SetVariable(ParCounter, ((*it)->GetParName(i)), (*it)->GetParInit(i), (*it)->GetDiagonalError(i)/10); - minuit->SetVariableValue(ParCounter, (*it)->GetParInit(i)); - //KS: lower bound, upper bound, if Mirroring enabled then ignore - if(!fMirroring) minuit->SetVariableLimits(ParCounter, (*it)->GetLowerBound(i), (*it)->GetUpperBound(i)); - if((*it)->IsParameterFixed(i)) - { - minuit->FixVariable(ParCounter); - } - } - } - else - { - for(int i = 0; i < (*it)->GetNParameters(); ++i, ++ParCounter) - { - minuit->SetVariable(ParCounter, Form("%i_PCA", i), (*it)->GetPCAHandler()->GetParPropPCA(i), (*it)->GetPCAHandler()->GetEigenValuesMaster()[i]/10); - if((*it)->GetPCAHandler()->IsParameterFixedPCA(i)) - { - minuit->FixVariable(ParCounter); - } + for (auto &parhandlr : systematics) { + + for (int i = 0; i < parhandlr->GetNumProposalParams(); ++i, ++ParCounter) { + // KS: Index, name, prior, step scale [different to MCMC], + minuit->SetVariable(ParCounter, (parhandlr->GetPCParName(i)), + parhandlr->GetPCParInit(i), + parhandlr->GetPCDiagonalError(i) / 10.0); + minuit->SetVariableValue(ParCounter, parhandlr->GetPCParInit(i)); + // KS: lower bound, upper bound, if Mirroring enabled then ignore + if (!fMirroring) + minuit->SetVariableLimits(ParCounter, parhandlr->GetPCLowerBound(i), + parhandlr->GetPCUpperBound(i)); + if (parhandlr->IsPCParameterFixed(i)) { + minuit->FixVariable(ParCounter); } } } @@ -100,102 +87,65 @@ void MinuitFit::RunMCMC() { MACH3LOG_INFO("Starting HESSE"); minuit->Hesse(); outputFile->cd(); - - TVectorD* MinuitParValue = new TVectorD(NparsMinuitFull); - TVectorD* MinuitParError = new TVectorD(NparsMinuitFull); - TMatrixDSym* Postmatrix = new TMatrixDSym(NparsMinuitFull); - for(int i = 0; i < NparsMinuitFull; ++i) - { + TVectorD *MinuitParValue = new TVectorD(NParsPCA); + TVectorD *MinuitParError = new TVectorD(NParsPCA); + TMatrixDSym *Postmatrix = new TMatrixDSym(NParsPCA); + + for (int i = 0; i < NParsPCA; ++i) { (*MinuitParValue)(i) = 0; (*MinuitParError)(i) = 0; - for(int j = 0; j < NparsMinuitFull; ++j) - { - (*Postmatrix)(i,j) = 0; - (*Postmatrix)(i,j) = minuit->CovMatrix(i,j); + for (int j = 0; j < NParsPCA; ++j) { + (*Postmatrix)(i, j) = 0; + (*Postmatrix)(i, j) = minuit->CovMatrix(i, j); } } ParCounter = 0; const double *X = minuit->X(); const double *err = minuit->Errors(); - for (std::vector::iterator it = systematics.begin(); it != systematics.end(); ++it) - { - if(!(*it)->IsPCA()) - { - for(int i = 0; i < (*it)->GetNumParams(); ++i, ++ParCounter) - { - double ParVal = X[ParCounter]; - //KS: Basically apply mirroring for parameters out of bounds - if(fMirroring) - { - if(ParVal < (*it)->GetLowerBound(i)) - { - ParVal = (*it)->GetLowerBound(i) + ((*it)->GetLowerBound(i) - ParVal); - } - else if (ParVal > (*it)->GetUpperBound(i)) - { - ParVal = (*it)->GetUpperBound(i) - ( ParVal - (*it)->GetUpperBound(i)); - } - } - (*MinuitParValue)(ParCounter) = ParVal; - (*MinuitParError)(ParCounter) = err[ParCounter]; - //KS: For fixed params HESS will not calculate error so we need to pass prior error - if((*it)->IsParameterFixed(i)) - { - (*MinuitParError)(ParCounter) = (*it)->GetDiagonalError(i); - (*Postmatrix)(ParCounter,ParCounter) = (*MinuitParError)(ParCounter) * (*MinuitParError)(ParCounter); + for (auto &parhandlr : systematics) { + + // set the parameters in the sampler basis + std::copy_n(X + ParCounter, parhandlr->GetNumProposalParams(), + parhandlr->proposer.params.proposed.data()); + + // this accepts the step for the proposer and rotates the parameters back to + // the systematic basis + parhandlr->AcceptStep(); + + for (int i = 0; i < parhandlr->GetNumSystematicParams(); ++i) { + double ParVal = parhandlr->GetParCurr(i); + // KS: Basically apply mirroring for parameters out of bounds + if (fMirroring) { // mirror in the systematic basis as it is where the + // bounds are defined + if (ParVal < parhandlr->GetLowerBound(i)) { + ParVal = parhandlr->GetLowerBound(i) + + (parhandlr->GetLowerBound(i) - ParVal); + } else if (ParVal > parhandlr->GetUpperBound(i)) { + ParVal = parhandlr->GetUpperBound(i) - + (ParVal - parhandlr->GetUpperBound(i)); } + parhandlr->SetParCurrProp(i, ParVal); } } - else - { - //KS: We need to convert parameters from PCA to normal base - TVectorD ParVals((*it)->GetNumParams()); - TVectorD ParVals_PCA((*it)->GetNParameters()); - - TVectorD ErrorVals((*it)->GetNumParams()); - TVectorD ErrorVals_PCA((*it)->GetNParameters()); - - TMatrixD MatrixVals((*it)->GetNumParams(), (*it)->GetNumParams()); - TMatrixD MatrixVals_PCA((*it)->GetNParameters(), (*it)->GetNParameters()); - - //First save them - //KS: This code is super convoluted as MaCh3 can store separate matrices while Minuit has one matrix. In future this will be simplified, keep it like this for now. - const int StartVal = ParCounter; - for(int i = 0; i < (*it)->GetNParameters(); ++i, ++ParCounter) - { - ParVals_PCA(i) = X[ParCounter]; - ErrorVals_PCA(i) = err[ParCounter]; - int ParCounterMatrix = StartVal; - for(int j = 0; j < (*it)->GetNParameters(); ++j, ++ParCounterMatrix) - { - MatrixVals_PCA(i,j) = minuit->CovMatrix(ParCounter,ParCounterMatrix); - } - } - ParVals = ((*it)->GetPCAHandler()->GetTransferMatrix())*ParVals_PCA; - ErrorVals = ((*it)->GetPCAHandler()->GetTransferMatrix())*ErrorVals_PCA; - MatrixVals.Mult(((*it)->GetPCAHandler()->GetTransferMatrix()),MatrixVals_PCA); - - ParCounter = StartVal; - //KS: Now after going from PCA to normal let';s save it - for(int i = 0; i < (*it)->GetNumParams(); ++i, ++ParCounter) - { - (*MinuitParValue)(ParCounter) = ParVals(i); - (*MinuitParError)(ParCounter) = std::fabs(ErrorVals(i)); - int ParCounterMatrix = StartVal; - for(int j = 0; j < (*it)->GetNumParams(); ++j, ++ParCounterMatrix) - { - (*Postmatrix)(ParCounter,ParCounterMatrix) = MatrixVals(i,j); - } - //If fixed take prior - if((*it)->GetPCAHandler()->IsParameterFixedPCA(i)) - { - (*MinuitParError)(ParCounter) = (*it)->GetDiagonalError(i); - (*Postmatrix)(ParCounter,ParCounter) = (*MinuitParError)(ParCounter) * (*MinuitParError)(ParCounter); - } + + for (int i = 0; i < parhandlr->GetNumProposalParams(); ++i) { + + (*MinuitParValue)(ParCounter + i) = + parhandlr->proposer.params.proposed[i]; + (*MinuitParError)(ParCounter + i) = err[ParCounter + i]; + + // KS: For fixed params HESS will not calculate error so we need to pass + // prior error + if (parhandlr->IsPCParameterFixed(i)) { + (*MinuitParError)(ParCounter) = parhandlr->GetPCDiagonalError(i); + (*Postmatrix)(ParCounter, ParCounter) = + (*MinuitParError)(ParCounter) * (*MinuitParError)(ParCounter); } } + + ParCounter += parhandlr->GetNumProposalParams(); } MinuitParValue->Write("MinuitParValue"); @@ -207,4 +157,3 @@ void MinuitFit::RunMCMC() { // Save all the output SaveOutput(); } - diff --git a/Fitters/PSO.cpp b/Fitters/PSO.cpp index 128aaedd7..8d6506705 100644 --- a/Fitters/PSO.cpp +++ b/Fitters/PSO.cpp @@ -4,40 +4,51 @@ // *************** PSO::PSO(manager *man) : LikelihoodFit(man) { -// *************** + // *************** AlgorithmName = "PSO"; - fConstriction = Get(fitMan->raw()["General"]["PSO"]["Constriction"], __FILE__, __LINE__); - fInertia = Get(fitMan->raw()["General"]["PSO"]["Inertia"], __FILE__, __LINE__) * fConstriction; - fOne = Get(fitMan->raw()["General"]["PSO"]["One"], __FILE__, __LINE__) * fConstriction; - fTwo = Get(fitMan->raw()["General"]["PSO"]["Two"], __FILE__, __LINE__) * fConstriction; - fParticles = Get(fitMan->raw()["General"]["PSO"]["Particles"], __FILE__, __LINE__); - fIterations = Get(fitMan->raw()["General"]["PSO"]["Iterations"], __FILE__, __LINE__); - fConvergence = Get(fitMan->raw()["General"]["PSO"]["Convergence"], __FILE__, __LINE__); + fConstriction = Get(fitMan->raw()["General"]["PSO"]["Constriction"], + __FILE__, __LINE__); + fInertia = Get(fitMan->raw()["General"]["PSO"]["Inertia"], __FILE__, + __LINE__) * + fConstriction; + fOne = + Get(fitMan->raw()["General"]["PSO"]["One"], __FILE__, __LINE__) * + fConstriction; + fTwo = + Get(fitMan->raw()["General"]["PSO"]["Two"], __FILE__, __LINE__) * + fConstriction; + fParticles = Get(fitMan->raw()["General"]["PSO"]["Particles"], __FILE__, + __LINE__); + fIterations = Get(fitMan->raw()["General"]["PSO"]["Iterations"], + __FILE__, __LINE__); + fConvergence = Get(fitMan->raw()["General"]["PSO"]["Convergence"], + __FILE__, __LINE__); fDim = 0; - if(fTestLikelihood) - { - fDim = Get(fitMan->raw()["General"]["PSO"]["TestLikelihoodDim"], __FILE__, __LINE__); + if (fTestLikelihood) { + fDim = Get(fitMan->raw()["General"]["PSO"]["TestLikelihoodDim"], + __FILE__, __LINE__); } } // *************** -void PSO::RunMCMC(){ -// *************** +void PSO::RunMCMC() { + // *************** PrepareFit(); // Remove obsolete memory and make other checks before fit starts SanitiseInputs(); - if(fTestLikelihood){ + if (fTestLikelihood) { outTree->Branch("nParts", &fParticles, "nParts/I"); - for(int i = 0; i < fDim; ++i){ + for (int i = 0; i < fDim; ++i) { par = new double[fParticles]; paramlist.push_back(par); - outTree->Branch(Form("Parameter_%d", i), paramlist[i], Form("Parameter_%d[nParts]/D",i)); + outTree->Branch(Form("Parameter_%d", i), paramlist[i], + Form("Parameter_%d[nParts]/D", i)); } -// vel = new double[fParticles]; + // vel = new double[fParticles]; outTree->Branch("vel", vel, "vel[nParts]/D"); } @@ -48,100 +59,82 @@ void PSO::RunMCMC(){ } // ************************* -void PSO::init(){ -// ************************* +void PSO::init() { + // ************************* fBestValue = M3::_LARGE_LOGL_; - //KS: For none PCA this will be eqaul to normal parameters - //const int NparsPSOFull = NPars; - //const int NparsPSO = NParsPCA; + // KS: For none PCA this will be eqaul to normal parameters + // const int NparsPSOFull = NPars; + // const int NparsPSO = NParsPCA; MACH3LOG_INFO("Preparing PSO"); // Initialise bounds on parameters - if(fTestLikelihood){ - for (int i = 0; i < fDim; i++){ + if (fTestLikelihood) { + for (int i = 0; i < fDim; i++) { // Test function ranges ranges_min.push_back(-5); ranges_max.push_back(5); fixed.push_back(0); } - } - else{ - for (std::vector::iterator it = systematics.begin(); it != systematics.end(); ++it){ - if(!(*it)->IsPCA()) - { - fDim += (*it)->GetNumParams(); - for(int i = 0; i < (*it)->GetNumParams(); ++i) - { - double curr = (*it)->GetParInit(i); - double lim = 10.0*(*it)->GetDiagonalError(i); - double low = (*it)->GetLowerBound(i); - double high = (*it)->GetUpperBound(i); - if(low > curr - lim) ranges_min.push_back(low); - else ranges_min.push_back(curr - lim); - if(high < curr + lim) ranges_min.push_back(high); - else ranges_min.push_back(curr + lim); - prior.push_back(curr); - - if((*it)->IsParameterFixed(i)){ - fixed.push_back(1); - } - else{ - fixed.push_back(0); - } - } - } - else - { - fDim += (*it)->GetNParameters(); - for(int i = 0; i < (*it)->GetNParameters(); ++i) - { - ranges_min.push_back(-100.0); - ranges_max.push_back(100.0); - prior.push_back((*it)->GetParInit(i)); - if((*it)->GetPCAHandler()->IsParameterFixedPCA(i)){ - fixed.push_back(1); - } - else{ - fixed.push_back(0); - } + } else { + for (auto &parhandlr : systematics) { + fDim += parhandlr->GetNumProposalParams(); + for (int i = 0; i < parhandlr->GetNumProposalParams(); ++i) { + double curr = parhandlr->GetPCParInit(i); + double lim = 10.0 * parhandlr->GetPCDiagonalError(i); + double low = parhandlr->GetPCLowerBound(i); + double high = parhandlr->GetPCUpperBound(i); + if (low > curr - lim) + ranges_min.push_back(low); + else + ranges_min.push_back(curr - lim); + if (high < curr + lim) + ranges_min.push_back(high); + else + ranges_min.push_back(curr + lim); + prior.push_back(curr); + + if (parhandlr->IsPCParameterFixed(i)) { + fixed.push_back(1); + } else { + fixed.push_back(0); } } } } MACH3LOG_INFO("Printing Minimums and Maximums of Variables to be minimized"); - for (int i = 0; i < fDim; i++){ - MACH3LOG_INFO("Variable {} : {:.2f}, {:.2f}", i, ranges_min[i], ranges_max[i]); + for (int i = 0; i < fDim; i++) { + MACH3LOG_INFO("Variable {} : {:.2f}, {:.2f}", i, ranges_min[i], + ranges_max[i]); } // Initialise particle positions - for (int i = 0; i < fParticles; ++i){ + for (int i = 0; i < fParticles; ++i) { std::vector init_position; std::vector init_velocity; - //Initialising in +/- 5sigma of prior value from BANFF interface - for (int j=0; jRndm()*dist); - //Initialise velocity to random position uniform in space - init_velocity.push_back((2.0*random->Rndm()-1.0));//*dist); + } else { + double dist = fabs(ranges_max[j] - ranges_min[j]); + // Initialise to random position uniform in space + init_position.push_back(ranges_min[j] + random->Rndm() * dist); + // Initialise velocity to random position uniform in space + init_velocity.push_back((2.0 * random->Rndm() - 1.0)); //*dist); } } - particle* new_particle = new particle(init_position,init_velocity); + particle *new_particle = new particle(init_position, init_velocity); new_particle->set_personal_best_position(init_position); double new_value = CalcChi(init_position); new_particle->set_personal_best_value(new_value); new_particle->set_value(new_value); system.push_back(new_particle); - if(new_value < fBestValue){ + if (new_value < fBestValue) { fBestValue = new_value; set_best_particle(new_particle); } @@ -149,93 +142,112 @@ void PSO::init(){ } // ************************* -std::vector > PSO::bisection(const std::vector& position, const double minimum, - const double range, const double precision) { -// ************************* +std::vector> +PSO::bisection(const std::vector &position, const double minimum, + const double range, const double precision) { + // ************************* std::vector> uncertainties_list; - for (unsigned int i = 0; i< position.size(); ++i) { + for (unsigned int i = 0; i < position.size(); ++i) { MACH3LOG_INFO("{}", i); - //std::vector uncertainties; - std::vector new_position = position; new_position[i] = position[i]-range; - double val_1 = CalcChi(new_position)-minimum-1.0; - while (val_1*-1.0 > 0.0){ + // std::vector uncertainties; + std::vector new_position = position; + new_position[i] = position[i] - range; + double val_1 = CalcChi(new_position) - minimum - 1.0; + while (val_1 * -1.0 > 0.0) { new_position[i] -= range; - val_1 = CalcChi(new_position)-minimum-1.0; + val_1 = CalcChi(new_position) - minimum - 1.0; } - std::vector bisect_position = position; bisect_position[i] = bisect_position[i] - (position[i]-new_position[i])/2; - std::vector> position_list{new_position,bisect_position,position}; - double val_2 = CalcChi(bisect_position)-minimum-1.0; - std::vector value_list{val_1,val_2, -1.0}; + std::vector bisect_position = position; + bisect_position[i] = + bisect_position[i] - (position[i] - new_position[i]) / 2; + std::vector> position_list{new_position, + bisect_position, position}; + double val_2 = CalcChi(bisect_position) - minimum - 1.0; + std::vector value_list{val_1, val_2, -1.0}; double res = 1.0; - while (res > precision){ - if (value_list[0] * value_list[1] < 0){ + while (res > precision) { + if (value_list[0] * value_list[1] < 0) { std::vector new_bisect_position = position_list[0]; - new_bisect_position[i] =new_bisect_position[i]+ (position_list[1][i]-position_list[0][i])/2; - double new_val = CalcChi(new_bisect_position)-minimum-1.0; + new_bisect_position[i] = + new_bisect_position[i] + + (position_list[1][i] - position_list[0][i]) / 2; + double new_val = CalcChi(new_bisect_position) - minimum - 1.0; position_list[2] = position_list[1]; value_list[2] = value_list[1]; value_list[1] = new_val; position_list[1] = new_bisect_position; - res = std::abs(position[2]-position[0]); - } - else{ + res = std::abs(position[2] - position[0]); + } else { std::vector new_bisect_position = position_list[1]; - new_bisect_position[i] += (position_list[2][i]-position_list[1][i])/2; - double new_val = CalcChi(new_bisect_position)-minimum-1.0; + new_bisect_position[i] += + (position_list[2][i] - position_list[1][i]) / 2; + double new_val = CalcChi(new_bisect_position) - minimum - 1.0; position_list[0] = position_list[1]; value_list[0] = value_list[1]; value_list[1] = new_val; position_list[1] = new_bisect_position; - res = std::abs(position_list[2][i]-position_list[1][i]); + res = std::abs(position_list[2][i] - position_list[1][i]); } } - //do the same thing for position uncertainty - std::vector new_position_p = position; new_position_p[i] = position[i]+range; - double val_1_p = CalcChi(new_position_p)-minimum-1.0; - while (val_1_p * -1.0 > 0.0){ + // do the same thing for position uncertainty + std::vector new_position_p = position; + new_position_p[i] = position[i] + range; + double val_1_p = CalcChi(new_position_p) - minimum - 1.0; + while (val_1_p * -1.0 > 0.0) { new_position_p[i] += range; - val_1_p = CalcChi(new_position_p)-minimum-1.0; + val_1_p = CalcChi(new_position_p) - minimum - 1.0; } - std::vector bisect_position_p = position; bisect_position_p[i] = bisect_position_p[i] += (new_position_p[i]-position[i])/2; - std::vector> position_list_p{position,bisect_position_p,new_position_p}; - double val_2_p = CalcChi(bisect_position_p)-minimum-1.0; - std::vector value_list_p{-1.0,val_2_p, val_1_p}; + std::vector bisect_position_p = position; + bisect_position_p[i] = bisect_position_p[i] += + (new_position_p[i] - position[i]) / 2; + std::vector> position_list_p{ + position, bisect_position_p, new_position_p}; + double val_2_p = CalcChi(bisect_position_p) - minimum - 1.0; + std::vector value_list_p{-1.0, val_2_p, val_1_p}; double res_p = 1.0; - while (res_p > precision){ - if (value_list_p[0] * value_list_p[1] < 0){ - std::vector new_bisect_position_p = position_list_p[0];new_bisect_position_p[i] += (position_list_p[1][i]-position_list_p[0][i])/2; - double new_val_p = CalcChi(new_bisect_position_p)-minimum-1.0; + while (res_p > precision) { + if (value_list_p[0] * value_list_p[1] < 0) { + std::vector new_bisect_position_p = position_list_p[0]; + new_bisect_position_p[i] += + (position_list_p[1][i] - position_list_p[0][i]) / 2; + double new_val_p = CalcChi(new_bisect_position_p) - minimum - 1.0; position_list_p[2] = position_list_p[1]; value_list_p[2] = value_list_p[1]; value_list_p[1] = new_val_p; position_list_p[1] = new_bisect_position_p; - res = std::abs(position[2]-position[0]); - res_p = std::abs(position_list_p[1][i]-position_list_p[0][i]); + res = std::abs(position[2] - position[0]); + res_p = std::abs(position_list_p[1][i] - position_list_p[0][i]); MACH3LOG_TRACE("Pos midpoint is {:.2f}", position_list_p[1][i]); - } - else{ - std::vector new_bisect_position_p = position_list_p[1];new_bisect_position_p[i] += (position_list_p[2][i]-position_list_p[1][i])/2; - double new_val_p = CalcChi(new_bisect_position_p)-minimum-1.0; + } else { + std::vector new_bisect_position_p = position_list_p[1]; + new_bisect_position_p[i] += + (position_list_p[2][i] - position_list_p[1][i]) / 2; + double new_val_p = CalcChi(new_bisect_position_p) - minimum - 1.0; position_list_p[0] = position_list_p[1]; value_list_p[0] = value_list_p[1]; value_list_p[1] = new_val_p; position_list_p[1] = new_bisect_position_p; - res_p = std::abs(position_list_p[2][i]-position_list_p[1][i]); + res_p = std::abs(position_list_p[2][i] - position_list_p[1][i]); MACH3LOG_TRACE("Pos midpoint is {:.2f}", position_list_p[1][i]); } } - uncertainties_list.push_back({std::abs(position[i]-position_list[1][i]),std::abs(position[i]-position_list_p[1][i])}); + uncertainties_list.push_back( + {std::abs(position[i] - position_list[1][i]), + std::abs(position[i] - position_list_p[1][i])}); MACH3LOG_INFO("Uncertainty finished for d = {}", i); - MACH3LOG_INFO("LLR values for ± positive and negative uncertainties are {:<10.2f} and {:<10.2f}", + MACH3LOG_INFO("LLR values for ± positive and negative uncertainties are " + "{:<10.2f} and {:<10.2f}", CalcChi(position_list[1]), CalcChi(position_list_p[1])); } return uncertainties_list; } // ************************* -std::vector> PSO::calc_uncertainty(const std::vector& position, const double minimum) { -// ************************* +std::vector> +PSO::calc_uncertainty(const std::vector &position, + const double minimum) { + // ************************* std::vector pos_uncertainty(position.size()); std::vector neg_uncertainty(position.size()); constexpr int num = 200; @@ -248,7 +260,7 @@ std::vector> PSO::calc_uncertainty(const std::vector double start = position[i]; std::vector x(num); std::vector y(num); - double StepPoint = (start-neg_stop) / (num - 1); + double StepPoint = (start - neg_stop) / (num - 1); double value = start; for (int j = 0; j < num; ++j) { pos[i] = value; @@ -272,7 +284,7 @@ std::vector> PSO::calc_uncertainty(const std::vector MACH3LOG_INFO("Neg"); x.assign(num, 0); y.assign(num, 0); - StepPoint = (pos_stop-start) / (num - 1); + StepPoint = (pos_stop - start) / (num - 1); value = start; for (int j = 0; j < num; ++j) { pos[i] = value; @@ -293,18 +305,18 @@ std::vector> PSO::calc_uncertainty(const std::vector } pos_uncertainty[i] = x[closest_index]; } - std::vector> res{neg_uncertainty,pos_uncertainty}; + std::vector> res{neg_uncertainty, pos_uncertainty}; return res; } // ************************* -void PSO::uncertainty_check(const std::vector& previous_pos){ -// ************************* - std::vector> x_list; - std::vector> y_list; +void PSO::uncertainty_check(const std::vector &previous_pos) { + // ************************* + std::vector> x_list; + std::vector> y_list; std::vector position = previous_pos; constexpr int num = 5000; - for (unsigned int i = 0;i& previous_pos){ double StepPoint = (stop - start) / (num - 1); double value = start; MACH3LOG_TRACE("result for fDim: {}", 1); - for (int j = 0;j < num; ++j) { + for (int j = 0; j < num; ++j) { position[i] = value; double LLR = CalcChi(position); x[j] = value; @@ -322,13 +334,13 @@ void PSO::uncertainty_check(const std::vector& previous_pos){ } position[i] = curr_ival; MACH3LOG_INFO(""); - MACH3LOG_INFO("For fDim{} x list is", i+1); - for (unsigned int k = 0;k& previous_pos){ } // ************************* -double PSO::swarmIterate(){ -// ************************* - std::vector total_pos(fDim,0.0); +double PSO::swarmIterate() { + // ************************* + std::vector total_pos(fDim, 0.0); for (int i = 0; i < fParticles; ++i) { - std::vector part1 = vector_multiply(system[i]->get_velocity(), fInertia); - std::vector part2 = vector_multiply(vector_subtract(system[i]->get_personal_best_position(), system[i]->get_position()), (fOne * random->Rndm())); - std::vector part3 = vector_multiply(vector_subtract(get_best_particle()->get_personal_best_position(), system[i]->get_position()),(fTwo * random->Rndm())); - std::vector new_velocity = three_vector_addition(part1, part2, part3); - std::vector new_pos = vector_add(system[i]->get_position(), new_velocity); - transform(total_pos.begin(), total_pos.end(), new_pos.begin(), total_pos.begin(),[](double x, double y) {return x+y;}); + std::vector part1 = + vector_multiply(system[i]->get_velocity(), fInertia); + std::vector part2 = + vector_multiply(vector_subtract(system[i]->get_personal_best_position(), + system[i]->get_position()), + (fOne * random->Rndm())); + std::vector part3 = vector_multiply( + vector_subtract(get_best_particle()->get_personal_best_position(), + system[i]->get_position()), + (fTwo * random->Rndm())); + std::vector new_velocity = + three_vector_addition(part1, part2, part3); + std::vector new_pos = + vector_add(system[i]->get_position(), new_velocity); + transform(total_pos.begin(), total_pos.end(), new_pos.begin(), + total_pos.begin(), [](double x, double y) { return x + y; }); for (int j = 0; j < fDim; ++j) { // Check if out of bounds and reflect if so - if(ranges_min[j] > new_pos[j]){ - new_pos[j] = ranges_min[j]; - } - else if(new_pos[j] > ranges_max[j]) { - new_pos[j] = ranges_max[j]; + if (ranges_min[j] > new_pos[j]) { + new_pos[j] = ranges_min[j]; + } else if (new_pos[j] > ranges_max[j]) { + new_pos[j] = ranges_max[j]; } // If parameter fixed don't update it - if(fixed[j]) new_pos[j] = system[i]->get_position()[j]; + if (fixed[j]) + new_pos[j] = system[i]->get_position()[j]; } - if(fTestLikelihood){ + if (fTestLikelihood) { double velo = 0.0; for (int j = 0; j < fDim; ++j) { paramlist[j][i] = new_pos[j]; - velo += new_velocity[j]*new_velocity[j]; + velo += new_velocity[j] * new_velocity[j]; } vel[i] = sqrt(velo); } @@ -372,24 +394,27 @@ double PSO::swarmIterate(){ system[i]->set_velocity(new_velocity); system[i]->set_position(new_pos); double new_value = CalcChi(new_pos); - if(new_value <= system[i]->get_personal_best_value()) { + if (new_value <= system[i]->get_personal_best_value()) { system[i]->set_personal_best_value(new_value); system[i]->set_personal_best_position(new_pos); - if(new_value < fBestValue){ + if (new_value < fBestValue) { fBestValue = new_value; set_best_particle(system[i]); } } } - std::vector best_pos = get_best_particle()->get_personal_best_position(); + std::vector best_pos = + get_best_particle()->get_personal_best_position(); std::vector result(best_pos.size(), 0.0); - transform(total_pos.begin(), total_pos.end(), total_pos.begin(), [=](double val){return val/fParticles;}); - transform(total_pos.begin(),total_pos.end(),best_pos.begin(),result.begin(),[](double x, double y) {return x-y;}); + transform(total_pos.begin(), total_pos.end(), total_pos.begin(), + [=](double val) { return val / fParticles; }); + transform(total_pos.begin(), total_pos.end(), best_pos.begin(), + result.begin(), [](double x, double y) { return x - y; }); double mean_dist_sq = 0; - for (int i = 0; iRndm()+1.0)*0.5)*(10.0/meanVel); + // Weight inertia randomly but scaled by total distance of swarm from global + // minimum - proxy for total velocity fWeight = + // ((random->Rndm()+1.0)*0.5)*(10.0/meanVel); logLCurr = fBestValue; outTree->Fill(); // Auto save the output - if (step % auto_save == 0) outTree->AutoSave(); + if (step % auto_save == 0) + outTree->AutoSave(); step++; accCount++; - if (i%100 == 0){ + if (i % 100 == 0) { MACH3LOG_INFO("Mean Dist Sq = {:.2f}", mean_dist_sq); MACH3LOG_INFO("Current LLR = {:.2f}", fBestValue); MACH3LOG_INFO("Position:"); - for (int j = 0; j < fDim; ++j){ - MACH3LOG_INFO(" Dim {} = {:<10.2f}", j, get_best_particle()->get_personal_best_position()[j]); + for (int j = 0; j < fDim; ++j) { + MACH3LOG_INFO(" Dim {} = {:<10.2f}", j, + get_best_particle()->get_personal_best_position()[j]); } } - if(fConvergence > 0.0){ - if(mean_dist_sq < fConvergence){ + if (fConvergence > 0.0) { + if (mean_dist_sq < fConvergence) { break; } } } MACH3LOG_INFO("Finished after {} runs out of {}", iter, fIterations); MACH3LOG_INFO("Mean Dist: {:.2f}", mean_dist_sq); - MACH3LOG_INFO("Best LLR: {:.2f}", get_best_particle()->get_personal_best_value()); - uncertainties = bisection(get_best_particle()->get_personal_best_position(),get_best_particle()->get_personal_best_value(),0.5,0.005); + MACH3LOG_INFO("Best LLR: {:.2f}", + get_best_particle()->get_personal_best_value()); + uncertainties = + bisection(get_best_particle()->get_personal_best_position(), + get_best_particle()->get_personal_best_value(), 0.5, 0.005); MACH3LOG_INFO("Position for Global Minimum = "); - for (int i = 0; i< fDim; ++i){ - MACH3LOG_INFO(" Dim {} = {:<10.2f} +{:.2f}, -{:.2f}", i, get_best_particle()->get_personal_best_position()[i], uncertainties[i][1], uncertainties[i][0]); + for (int i = 0; i < fDim; ++i) { + MACH3LOG_INFO(" Dim {} = {:<10.2f} +{:.2f}, -{:.2f}", i, + get_best_particle()->get_personal_best_position()[i], + uncertainties[i][1], uncertainties[i][0]); } } // ************************* -void PSO::WriteOutput(){ -// ************************* +void PSO::WriteOutput() { + // ************************* outputFile->cd(); - TVectorD* PSOParValue = new TVectorD(fDim); - TVectorD* PSOParError = new TVectorD(fDim); + TVectorD *PSOParValue = new TVectorD(fDim); + TVectorD *PSOParError = new TVectorD(fDim); - for(int i = 0; i < fDim; ++i) - { + for (int i = 0; i < fDim; ++i) { (*PSOParValue)(i) = 0; (*PSOParError)(i) = 0; } - std::vector minimum = get_best_particle()->get_personal_best_position(); + std::vector minimum = + get_best_particle()->get_personal_best_position(); int ParCounter = 0; - if(fTestLikelihood){ - for(int i = 0; i < fDim; ++i){ + if (fTestLikelihood) { + for (int i = 0; i < fDim; ++i) { (*PSOParValue)(i) = minimum[i]; - (*PSOParError)(i) = (uncertainties[i][0]+uncertainties[i][1])/2.0; + (*PSOParError)(i) = (uncertainties[i][0] + uncertainties[i][1]) / 2.0; } - } - else{ - for (std::vector::iterator it = systematics.begin(); it != systematics.end(); ++it) - { - if(!(*it)->IsPCA()) + } else { + for (auto &parhandlr : systematics) { { - for(int i = 0; i < (*it)->GetNumParams(); ++i, ++ParCounter) - { + for (int i = 0; i < parhandlr->GetNumProposalParams(); + ++i, ++ParCounter) { double ParVal = minimum[ParCounter]; - //KS: Basically apply mirroring for parameters out of bounds + // KS: Basically apply mirroring for parameters out of bounds (*PSOParValue)(ParCounter) = ParVal; - (*PSOParError)(ParCounter) = (uncertainties[ParCounter][0]+uncertainties[ParCounter][1])/2.0; - //KS: For fixed params HESS will not calcuate error so we need to pass prior error - if((*it)->IsParameterFixed(i)) - { - (*PSOParError)(ParCounter) = (*it)->GetDiagonalError(i); - } - } - } - else - { - //KS: We need to convert parameters from PCA to normal base - TVectorD ParVals((*it)->GetNumParams()); - TVectorD ParVals_PCA((*it)->GetNParameters()); - - TVectorD ErrorVals((*it)->GetNumParams()); - TVectorD ErrorVals_PCA((*it)->GetNParameters()); - - //First save them - //KS: This code is super convoluted as MaCh3 can store separate matrices while PSO has one matrix. In future this will be simplified, keep it like this for now. - const int StartVal = ParCounter; - for(int i = 0; i < (*it)->GetNParameters(); ++i, ++ParCounter) - { - ParVals_PCA(i) = minimum[ParCounter]; - ErrorVals_PCA(i) = (uncertainties[ParCounter][0]+uncertainties[ParCounter][1])/2.0; - } - ParVals = ((*it)->GetPCAHandler()->GetTransferMatrix())*ParVals_PCA; - ErrorVals = ((*it)->GetPCAHandler()->GetTransferMatrix())*ErrorVals_PCA; - - ParCounter = StartVal; - //KS: Now after going from PCA to normal let';s save it - for(int i = 0; i < (*it)->GetNumParams(); ++i, ++ParCounter) - { - (*PSOParValue)(ParCounter) = ParVals(i); - (*PSOParError)(ParCounter) = std::fabs(ErrorVals(i)); - //int ParCounterMatrix = StartVal; - //If fixed take prior - if((*it)->GetPCAHandler()->IsParameterFixedPCA(i)) - { - (*PSOParError)(ParCounter) = (*it)->GetDiagonalError(i); + (*PSOParError)(ParCounter) = + (uncertainties[ParCounter][0] + uncertainties[ParCounter][1]) / + 2.0; + // KS: For fixed params HESS will not calcuate error so we need to + // pass prior error + if (parhandlr->IsParameterFixed(i)) { + (*PSOParError)(ParCounter) = parhandlr->GetDiagonalError(i); } } } @@ -528,25 +526,25 @@ void PSO::WriteOutput(){ } // ******************* -double PSO::CalcChi2(const double* x) { -// ******************* - if(fTestLikelihood) { +double PSO::CalcChi2(const double *x) { + // ******************* + if (fTestLikelihood) { return rastriginFunc(x); } else { - return LikelihoodFit::CalcChi2(x); + return LikelihoodFit::CalcChi2PC(x); } } // ************************* -double PSO::rastriginFunc(const double* x) { -// ************************* +double PSO::rastriginFunc(const double *x) { + // ************************* stepClock->Start(); - //Search range: [-5.12, 5.12] + // Search range: [-5.12, 5.12] constexpr double A = 10.0; double sum = 0.0; for (int i = 0; i < fDim; ++i) { - sum += x[i] * x[i] - A * cos(2.0 * 3.14 * x[i]); + sum += x[i] * x[i] - A * cos(2.0 * 3.14 * x[i]); } double llh = A * fDim + sum; diff --git a/Fitters/PredictiveThrower.cpp b/Fitters/PredictiveThrower.cpp index 1b80fd4c8..d33d37429 100644 --- a/Fitters/PredictiveThrower.cpp +++ b/Fitters/PredictiveThrower.cpp @@ -1,23 +1,24 @@ #include "PredictiveThrower.h" -#include "Samples/SampleHandlerFD.h" #include "Parameters/ParameterHandlerGeneric.h" #include "TH3.h" // ************************* PredictiveThrower::PredictiveThrower(manager *man) : FitterBase(man) { -// ************************* - AlgorithmName = "PredictiveThrower"; - if(!CheckNodeExists(fitMan->raw(), "Predictive")) { - MACH3LOG_ERROR("Predictive is missing in your main yaml config"); - throw MaCh3Exception(__FILE__ , __LINE__ ); + // ************************* + AlgorithmName = "PredictiveThrower"; + if (!CheckNodeExists(fitMan->raw(), "Predictive")) { + MACH3LOG_ERROR("Predictive is missing in your main yaml config"); + throw MaCh3Exception(__FILE__, __LINE__); } ModelSystematic = nullptr; // Use the full likelihood for the Prior/Posterior predictive pvalue - FullLLH = GetFromManager(fitMan->raw()["Predictive"]["FullLLH"], false, __FILE__, __LINE__ ); + FullLLH = GetFromManager(fitMan->raw()["Predictive"]["FullLLH"], false, + __FILE__, __LINE__); NModelParams = 0; - Is_PriorPredictive = Get(fitMan->raw()["Predictive"]["PriorPredictive"], __FILE__, __LINE__); + Is_PriorPredictive = Get(fitMan->raw()["Predictive"]["PriorPredictive"], + __FILE__, __LINE__); Ntoys = Get(fitMan->raw()["Predictive"]["Ntoy"], __FILE__, __LINE__); ReweightWeight.resize(Ntoys); @@ -27,15 +28,16 @@ PredictiveThrower::PredictiveThrower(manager *man) : FitterBase(man) { // ************************* // Destructor: PredictiveThrower::~PredictiveThrower() { -// ************************* - + // ************************* } // ************************* void PredictiveThrower::SetParamters() { -// ************************* + // ************************* // WARNING This should be removed in the future - auto DoNotThrowLegacyCov = GetFromManager>(fitMan->raw()["Predictive"]["DoNotThrowLegacyCov"], {}, __FILE__, __LINE__); + auto DoNotThrowLegacyCov = GetFromManager>( + fitMan->raw()["Predictive"]["DoNotThrowLegacyCov"], {}, __FILE__, + __LINE__); /// Have ability to not throw legacy matrices for (size_t i = 0; i < DoNotThrowLegacyCov.size(); ++i) { for (size_t s = 0; s < systematics.size(); ++s) { @@ -47,14 +49,15 @@ void PredictiveThrower::SetParamters() { } // Set groups to prefit values if they were set to not be varies - if(ModelSystematic && ParameterGroupsNotVaried.size() > 0) { + if (ModelSystematic && ParameterGroupsNotVaried.size() > 0) { ModelSystematic->SetGroupOnlyParameters(ParameterGroupsNotVaried); } /// Alternatively vary only selected params if (ModelSystematic && !ParameterOnlyToVary.empty()) { - for (int i = 0; i < ModelSystematic->GetNumParams(); ++i) { - // KS: If parameter is in map then we are skipping this, otherwise for params that we don't want to vary we simply set it to prior + for (int i = 0; i < ModelSystematic->GetNumSystematicParams(); ++i) { + // KS: If parameter is in map then we are skipping this, otherwise for + // params that we don't want to vary we simply set it to prior if (ParameterOnlyToVary.find(i) == ParameterOnlyToVary.end()) { ModelSystematic->SetParProp(i, ModelSystematic->GetParInit(i)); } @@ -64,13 +67,14 @@ void PredictiveThrower::SetParamters() { // ************************* void PredictiveThrower::SetupSampleInformation() { -// ************************* + // ************************* TotalNumberOfSamples = 0; - for (size_t iPDF = 0; iPDF < samples.size(); iPDF++) - { - auto* MaCh3Sample = dynamic_cast(samples[iPDF]); + for (size_t iPDF = 0; iPDF < samples.size(); iPDF++) { + auto *MaCh3Sample = dynamic_cast(samples[iPDF]); if (!MaCh3Sample) { - MACH3LOG_ERROR("Sample {} do not inherit from SampleHandlerFD this is not implemented", samples[iPDF]->GetName()); + MACH3LOG_ERROR("Sample {} do not inherit from SampleHandlerFD this is " + "not implemented", + samples[iPDF]->GetName()); throw MaCh3Exception(__FILE__, __LINE__); } TotalNumberOfSamples += samples[iPDF]->GetNsamples(); @@ -83,12 +87,14 @@ void PredictiveThrower::SetupSampleInformation() { W2_Nom_Hist.resize(TotalNumberOfSamples); SampleObjectMap.resize(TotalNumberOfSamples); - SampleNames.resize(TotalNumberOfSamples+1); + SampleNames.resize(TotalNumberOfSamples + 1); int currentIndex = 0; for (size_t iPDF = 0; iPDF < samples.size(); ++iPDF) { - for (int subSampleIndex = 0; subSampleIndex < samples[iPDF]->GetNsamples(); ++subSampleIndex) { - SampleObjectMap[currentIndex] = static_cast(iPDF); // map the current global sample index to this sample object + for (int subSampleIndex = 0; subSampleIndex < samples[iPDF]->GetNsamples(); + ++subSampleIndex) { + SampleObjectMap[currentIndex] = static_cast( + iPDF); // map the current global sample index to this sample object ++currentIndex; } } @@ -99,7 +105,8 @@ void PredictiveThrower::SetupSampleInformation() { } int counter = 0; for (size_t iPDF = 0; iPDF < samples.size(); iPDF++) { - for (int SampleIndex = 0; SampleIndex < samples[iPDF]->GetNsamples(); ++SampleIndex) { + for (int SampleIndex = 0; SampleIndex < samples[iPDF]->GetNsamples(); + ++SampleIndex) { SampleNames[counter] = samples[iPDF]->GetSampleTitle(SampleIndex); counter++; } @@ -110,11 +117,11 @@ void PredictiveThrower::SetupSampleInformation() { // ************************* // Produce MaCh3 toys: void PredictiveThrower::SetupToyGeneration() { -// ************************* + // ************************* int counter = 0; for (size_t s = 0; s < systematics.size(); ++s) { - auto* MaCh3Params = dynamic_cast(systematics[s]); - if(MaCh3Params) { + auto *MaCh3Params = dynamic_cast(systematics[s]); + if (MaCh3Params) { ModelSystematic = MaCh3Params; counter++; } @@ -122,65 +129,82 @@ void PredictiveThrower::SetupToyGeneration() { SetupSampleInformation(); - if(Is_PriorPredictive) { + if (Is_PriorPredictive) { MACH3LOG_INFO("You've chosen to run Prior Predictive Distribution"); } else { - auto PosteriorFileName = Get(fitMan->raw()["Predictive"]["PosteriorFile"], __FILE__, __LINE__); - //KS: We use MCMCProcessor to get names of covariances that were actually used to produce given chain + auto PosteriorFileName = Get( + fitMan->raw()["Predictive"]["PosteriorFile"], __FILE__, __LINE__); + // KS: We use MCMCProcessor to get names of covariances that were actually + // used to produce given chain MCMCProcessor Processor(PosteriorFileName); Processor.Initialise(); - ///Let's ask the manager what are the file with covariance matrix + /// Let's ask the manager what are the file with covariance matrix YAML::Node ConfigInChain = Processor.GetCovConfig(kXSecPar); - if(ModelSystematic) { + if (ModelSystematic) { YAML::Node ConfigNow = ModelSystematic->GetConfig(); - if (!compareYAMLNodes(ConfigNow, ConfigInChain)) - { - MACH3LOG_ERROR("Yaml configs used for your ParameterHandler for chain you want sample from ({}) and one currently initialised are different", PosteriorFileName); - throw MaCh3Exception(__FILE__ , __LINE__ ); + if (!compareYAMLNodes(ConfigNow, ConfigInChain)) { + MACH3LOG_ERROR( + "Yaml configs used for your ParameterHandler for chain you want " + "sample from ({}) and one currently initialised are different", + PosteriorFileName); + throw MaCh3Exception(__FILE__, __LINE__); } } } - if(counter > 1) { - MACH3LOG_ERROR("Found {} ParmaterHandler inheriting from ParameterHandlerGeneric, I can accept at most 1", counter); + if (counter > 1) { + MACH3LOG_ERROR("Found {} ParmaterHandler inheriting from " + "ParameterHandlerGeneric, I can accept at most 1", + counter); throw MaCh3Exception(__FILE__, __LINE__); } for (size_t s = 0; s < systematics.size(); ++s) { - NModelParams += systematics[s]->GetNumParams(); + NModelParams += systematics[s]->GetNumSystematicParams(); } if (ModelSystematic) { - auto ThrowParamGroupOnly = GetFromManager>(fitMan->raw()["Predictive"]["ThrowParamGroupOnly"], {}, __FILE__, __LINE__); + auto ThrowParamGroupOnly = GetFromManager>( + fitMan->raw()["Predictive"]["ThrowParamGroupOnly"], {}, __FILE__, + __LINE__); auto UniqueParamGroup = ModelSystematic->GetUniqueParameterGroups(); - auto ParameterOnlyToVaryString = GetFromManager>(fitMan->raw()["Predictive"]["ThrowSinlgeParams"], {}, __FILE__, __LINE__); + auto ParameterOnlyToVaryString = GetFromManager>( + fitMan->raw()["Predictive"]["ThrowSinlgeParams"], {}, __FILE__, + __LINE__); if (!ThrowParamGroupOnly.empty() && !ParameterOnlyToVaryString.empty()) { - MACH3LOG_ERROR("Can't use ThrowParamGroupOnly and ThrowSinlgeParams at the same time"); + MACH3LOG_ERROR("Can't use ThrowParamGroupOnly and ThrowSinlgeParams at " + "the same time"); throw MaCh3Exception(__FILE__, __LINE__); } if (!ParameterOnlyToVaryString.empty()) { - MACH3LOG_INFO("I will throw only: {}", fmt::join(ParameterOnlyToVaryString, ", ")); + MACH3LOG_INFO("I will throw only: {}", + fmt::join(ParameterOnlyToVaryString, ", ")); std::vector ParameterVary(ParameterOnlyToVaryString.size()); for (size_t i = 0; i < ParameterOnlyToVaryString.size(); ++i) { - ParameterVary[i] = ModelSystematic->GetParIndex(ParameterOnlyToVaryString[i]); + ParameterVary[i] = + ModelSystematic->GetParIndex(ParameterOnlyToVaryString[i]); if (ParameterVary[i] == M3::_BAD_INT_) { - MACH3LOG_ERROR("Can't proceed if param {} is missing", ParameterOnlyToVaryString[i]); + MACH3LOG_ERROR("Can't proceed if param {} is missing", + ParameterOnlyToVaryString[i]); throw MaCh3Exception(__FILE__, __LINE__); } } - ParameterOnlyToVary = std::unordered_set(ParameterVary.begin(), ParameterVary.end()); + ParameterOnlyToVary = + std::unordered_set(ParameterVary.begin(), ParameterVary.end()); } else { - MACH3LOG_INFO("I have following parameter groups: {}", fmt::join(UniqueParamGroup, ", ")); + MACH3LOG_INFO("I have following parameter groups: {}", + fmt::join(UniqueParamGroup, ", ")); if (ThrowParamGroupOnly.empty()) { MACH3LOG_INFO("I will vary all"); } else { - std::unordered_set throwOnlySet(ThrowParamGroupOnly.begin(), ThrowParamGroupOnly.end()); + std::unordered_set throwOnlySet( + ThrowParamGroupOnly.begin(), ThrowParamGroupOnly.end()); ParameterGroupsNotVaried.clear(); - for (const auto& group : UniqueParamGroup) { + for (const auto &group : UniqueParamGroup) { if (throwOnlySet.find(group) == throwOnlySet.end()) { ParameterGroupsNotVaried.push_back(group); } @@ -196,15 +220,16 @@ void PredictiveThrower::SetupToyGeneration() { // ************************* // Try loading toys bool PredictiveThrower::LoadToys() { -// ************************* - auto PosteriorFileName = Get(fitMan->raw()["Predictive"]["PosteriorFile"], __FILE__, __LINE__); + // ************************* + auto PosteriorFileName = Get( + fitMan->raw()["Predictive"]["PosteriorFile"], __FILE__, __LINE__); // Open the ROOT file int originalErrorWarning = gErrorIgnoreLevel; gErrorIgnoreLevel = kFatal; - TFile* file = TFile::Open(PosteriorFileName.c_str(), "READ"); + TFile *file = TFile::Open(PosteriorFileName.c_str(), "READ"); gErrorIgnoreLevel = originalErrorWarning; - TDirectory* ToyDir = nullptr; + TDirectory *ToyDir = nullptr; if (!file || file->IsZombie()) { return false; } else { @@ -218,8 +243,9 @@ bool PredictiveThrower::LoadToys() { } } - // Finally get the TTree branch with the penalty vectors for each of the toy throws - TTree* PenaltyTree = static_cast(file->Get("ToySummary")); + // Finally get the TTree branch with the penalty vectors for each of the toy + // throws + TTree *PenaltyTree = static_cast(file->Get("ToySummary")); if (!PenaltyTree) { MACH3LOG_WARN("ToySummary TTree not found in file."); file->Close(); @@ -228,9 +254,12 @@ bool PredictiveThrower::LoadToys() { } Ntoys = static_cast(PenaltyTree->GetEntries()); - int ConfigNtoys = Get(fitMan->raw()["Predictive"]["Ntoy"], __FILE__, __LINE__);; + int ConfigNtoys = + Get(fitMan->raw()["Predictive"]["Ntoy"], __FILE__, __LINE__); + ; if (Ntoys != ConfigNtoys) { - MACH3LOG_WARN("Found different number of toys in saved file than asked to run!"); + MACH3LOG_WARN( + "Found different number of toys in saved file than asked to run!"); MACH3LOG_INFO("Will read _ALL_ toys in the file"); MACH3LOG_INFO("Ntoys in file: {}", Ntoys); MACH3LOG_INFO("Ntoys specified: {}", ConfigNtoys); @@ -258,24 +287,28 @@ bool PredictiveThrower::LoadToys() { SetupSampleInformation(); for (int sample = 0; sample < TotalNumberOfSamples; ++sample) { - TH1* DataHist1D = static_cast(ToyDir->Get((SampleNames[sample] + "_data").c_str())); + TH1 *DataHist1D = static_cast( + ToyDir->Get((SampleNames[sample] + "_data").c_str())); Data_Hist[sample] = M3::Clone(DataHist1D); - TH1* MCHist1D = static_cast(ToyDir->Get((SampleNames[sample] + "_mc").c_str())); + TH1 *MCHist1D = + static_cast(ToyDir->Get((SampleNames[sample] + "_mc").c_str())); MC_Nom_Hist[sample] = M3::Clone(MCHist1D); - TH1* W2Hist1D = static_cast(ToyDir->Get((SampleNames[sample] + "_w2").c_str())); + TH1 *W2Hist1D = + static_cast(ToyDir->Get((SampleNames[sample] + "_w2").c_str())); W2_Nom_Hist[sample] = M3::Clone(W2Hist1D); } - - for (int iToy = 0; iToy < Ntoys; ++iToy) - { - if (iToy % 100 == 0) MACH3LOG_INFO(" Loaded toy {}", iToy); + for (int iToy = 0; iToy < Ntoys; ++iToy) { + if (iToy % 100 == 0) + MACH3LOG_INFO(" Loaded toy {}", iToy); for (int sample = 0; sample < TotalNumberOfSamples; ++sample) { - TH1* MCHist1D = static_cast(ToyDir->Get((SampleNames[sample] + "_mc_" + std::to_string(iToy)).c_str())); - TH1* W2Hist1D = static_cast(ToyDir->Get((SampleNames[sample] + "_w2_" + std::to_string(iToy)).c_str())); + TH1 *MCHist1D = static_cast(ToyDir->Get( + (SampleNames[sample] + "_mc_" + std::to_string(iToy)).c_str())); + TH1 *W2Hist1D = static_cast(ToyDir->Get( + (SampleNames[sample] + "_w2_" + std::to_string(iToy)).c_str())); MC_Hist_Toy[sample][iToy] = M3::Clone(MCHist1D); W2_Hist_Toy[sample][iToy] = M3::Clone(W2Hist1D); @@ -290,14 +323,16 @@ bool PredictiveThrower::LoadToys() { // ************************* // Produce MaCh3 toys: void PredictiveThrower::ProduceToys() { -// ************************* + // ************************* /// If we found toys then skip process of making new toys - if(LoadToys()) return; + if (LoadToys()) + return; /// Setup useful information for toy generation SetupToyGeneration(); - auto PosteriorFileName = Get(fitMan->raw()["Predictive"]["PosteriorFile"], __FILE__, __LINE__); + auto PosteriorFileName = Get( + fitMan->raw()["Predictive"]["PosteriorFile"], __FILE__, __LINE__); MACH3LOG_INFO("Starting {}", __func__); @@ -313,38 +348,45 @@ void PredictiveThrower::ProduceToys() { // KS: define branches so we can keep track of what params we are throwing std::vector ParamValues(NModelParams); - std::vector ParampPointers(NModelParams); + std::vector ParampPointers(NModelParams); int ParamCounter = 0; - for (size_t iSys = 0; iSys < systematics.size(); iSys++) - { - for (int iPar = 0; iPar < systematics[iSys]->GetNumParams(); iPar++) - { + for (size_t iSys = 0; iSys < systematics.size(); iSys++) { + for (int iPar = 0; iPar < systematics[iSys]->GetNumParams(); iPar++) { ParampPointers[ParamCounter] = systematics[iSys]->RetPointer(iPar); std::string Name = systematics[iSys]->GetParName(iPar); - ToyTree->Branch(Name.c_str(), &ParamValues[ParamCounter], (Name + "/D").c_str()); + ToyTree->Branch(Name.c_str(), &ParamValues[ParamCounter], + (Name + "/D").c_str()); ParamCounter++; } } - TDirectory* ToyDirectory = outputFile->mkdir("Toys"); + TDirectory *ToyDirectory = outputFile->mkdir("Toys"); ToyDirectory->cd(); int SampleCounter = 0; - for (size_t iPDF = 0; iPDF < samples.size(); iPDF++) - { - auto* MaCh3Sample = dynamic_cast(samples[iPDF]); - for (int SampleIndex = 0; SampleIndex < MaCh3Sample->GetNsamples(); ++SampleIndex) - { + for (size_t iPDF = 0; iPDF < samples.size(); iPDF++) { + auto *MaCh3Sample = dynamic_cast(samples[iPDF]); + for (int SampleIndex = 0; SampleIndex < MaCh3Sample->GetNsamples(); + ++SampleIndex) { // Get nominal spectra and event rates - TH1* DataHist1D = MaCh3Sample->GetDataHist(SampleIndex, MaCh3Sample->GetNDim(SampleIndex)); - Data_Hist[SampleCounter] = M3::Clone(DataHist1D, MaCh3Sample->GetSampleTitle(SampleIndex) + "_data"); - Data_Hist[SampleCounter]->Write((MaCh3Sample->GetSampleTitle(SampleIndex) + "_data").c_str()); - - TH1* MCHist1D = MaCh3Sample->GetMCHist(SampleIndex, MaCh3Sample->GetNDim(SampleIndex)); - MC_Nom_Hist[SampleCounter] = M3::Clone(MCHist1D, MaCh3Sample->GetSampleTitle(SampleIndex) + "_mc"); - MC_Nom_Hist[SampleCounter]->Write((MaCh3Sample->GetSampleTitle(SampleIndex) + "_mc").c_str()); - - TH1* W2Hist1D = MaCh3Sample->GetW2Hist(SampleIndex, MaCh3Sample->GetNDim(SampleIndex)); - W2_Nom_Hist[SampleCounter] = M3::Clone(W2Hist1D, MaCh3Sample->GetSampleTitle(SampleIndex) + "_w2"); - W2_Nom_Hist[SampleCounter]->Write((MaCh3Sample->GetSampleTitle(SampleIndex) + "_w2").c_str()); + TH1 *DataHist1D = MaCh3Sample->GetDataHist( + SampleIndex, MaCh3Sample->GetNDim(SampleIndex)); + Data_Hist[SampleCounter] = M3::Clone( + DataHist1D, MaCh3Sample->GetSampleTitle(SampleIndex) + "_data"); + Data_Hist[SampleCounter]->Write( + (MaCh3Sample->GetSampleTitle(SampleIndex) + "_data").c_str()); + + TH1 *MCHist1D = MaCh3Sample->GetMCHist(SampleIndex, + MaCh3Sample->GetNDim(SampleIndex)); + MC_Nom_Hist[SampleCounter] = + M3::Clone(MCHist1D, MaCh3Sample->GetSampleTitle(SampleIndex) + "_mc"); + MC_Nom_Hist[SampleCounter]->Write( + (MaCh3Sample->GetSampleTitle(SampleIndex) + "_mc").c_str()); + + TH1 *W2Hist1D = MaCh3Sample->GetW2Hist(SampleIndex, + MaCh3Sample->GetNDim(SampleIndex)); + W2_Nom_Hist[SampleCounter] = + M3::Clone(W2Hist1D, MaCh3Sample->GetSampleTitle(SampleIndex) + "_w2"); + W2_Nom_Hist[SampleCounter]->Write( + (MaCh3Sample->GetSampleTitle(SampleIndex) + "_w2").c_str()); delete W2Hist1D; SampleCounter++; } @@ -354,12 +396,11 @@ void PredictiveThrower::ProduceToys() { std::vector> branch_vals(systematics.size()); std::vector> branch_name(systematics.size()); - TChain* PosteriorFile = nullptr; + TChain *PosteriorFile = nullptr; unsigned int burn_in = 0; unsigned int maxNsteps = 0; unsigned int Step = 0; - if(!Is_PriorPredictive) - { + if (!Is_PriorPredictive) { PosteriorFile = new TChain("posteriors"); PosteriorFile->Add(PosteriorFileName.c_str()); @@ -373,64 +414,70 @@ void PredictiveThrower::ProduceToys() { } for (size_t s = 0; s < systematics.size(); ++s) { - systematics[s]->MatchMaCh3OutputBranches(PosteriorFile, branch_vals[s], branch_name[s]); + systematics[s]->MatchMaCh3OutputBranches(PosteriorFile, branch_vals[s], + branch_name[s]); } - //Get the burn-in from the config - burn_in = Get(fitMan->raw()["Predictive"]["BurnInSteps"], __FILE__, __LINE__); + // Get the burn-in from the config + burn_in = Get(fitMan->raw()["Predictive"]["BurnInSteps"], + __FILE__, __LINE__); - //DL: Adding sanity check for chains shorter than burn in + // DL: Adding sanity check for chains shorter than burn in maxNsteps = static_cast(PosteriorFile->GetMaximum("step")); - if(burn_in >= maxNsteps) - { + if (burn_in >= maxNsteps) { MACH3LOG_ERROR("You are running on a chain shorter than burn in cut"); - MACH3LOG_ERROR("Maximal value of nSteps: {}, burn in cut {}", maxNsteps, burn_in); + MACH3LOG_ERROR("Maximal value of nSteps: {}, burn in cut {}", maxNsteps, + burn_in); MACH3LOG_ERROR("You will run into infinite loop"); MACH3LOG_ERROR("You can make new chain or modify burn in cut"); - throw MaCh3Exception(__FILE__,__LINE__); + throw MaCh3Exception(__FILE__, __LINE__); } } TStopwatch TempClock; TempClock.Start(); - for(int i = 0; i < Ntoys; i++) - { - if( i % (Ntoys/10) == 0) { + for (int i = 0; i < Ntoys; i++) { + if (i % (Ntoys / 10) == 0) { MaCh3Utils::PrintProgressBar(i, Ntoys); } - if(!Is_PriorPredictive){ + if (!Is_PriorPredictive) { int entry = 0; Step = 0; - //YSP: Ensures you get an entry from the mcmc even when burn_in is set to zero (Although not advised :p ). - //Take 200k burn in steps, WP: Eb C in 1st peaky - // If we have combined chains by hadd need to check the step in the chain - // Note, entry is not necessarily same as step due to merged ROOT files, so can't choose entry in the range BurnIn - nEntries :( - while(Step < burn_in){ - entry = random->Integer(static_cast(PosteriorFile->GetEntries())); + // YSP: Ensures you get an entry from the mcmc even when burn_in is set to + // zero (Although not advised :p ). Take 200k burn in steps, WP: Eb C in + // 1st peaky + // If we have combined chains by hadd need to check the step in the chain + // Note, entry is not necessarily same as step due to merged ROOT files, + // so can't choose entry in the range BurnIn - nEntries :( + while (Step < burn_in) { + entry = random->Integer( + static_cast(PosteriorFile->GetEntries())); PosteriorFile->GetEntry(entry); } Draw = entry; } - for (size_t s = 0; s < systematics.size(); ++s) - { - //KS: Below line can help you get prior predictive distributions which are helpful for getting pre and post ND fit spectra - //YSP: If not set in the config, the code runs SK Posterior Predictive distributions by default. If true, then the code runs SK prior predictive. - if(Is_PriorPredictive) { + for (size_t s = 0; s < systematics.size(); ++s) { + // KS: Below line can help you get prior predictive distributions which + // are helpful for getting pre and post ND fit spectra YSP: If not set in + // the config, the code runs SK Posterior Predictive distributions by + // default. If true, then the code runs SK prior predictive. + if (Is_PriorPredictive) { systematics[s]->ThrowParameters(); } else { systematics[s]->SetParameters(branch_vals[s]); } } - // This set some params to prior value this way you can evaluate errors from subset of errors + // This set some params to prior value this way you can evaluate errors from + // subset of errors SetParamters(); Penalty = 0; - if(FullLLH) { + if (FullLLH) { for (size_t s = 0; s < systematics.size(); ++s) { - //KS: do times 2 because banff reports chi2 + // KS: do times 2 because banff reports chi2 Penalty = 2.0 * systematics[s]->GetLikelihood(); } } @@ -443,17 +490,22 @@ void PredictiveThrower::ProduceToys() { } SampleCounter = 0; - for (size_t iPDF = 0; iPDF < samples.size(); iPDF++) - { - auto* MaCh3Sample = dynamic_cast(samples[iPDF]); - for (int SampleIndex = 0; SampleIndex < MaCh3Sample->GetNsamples(); ++SampleIndex) - { - TH1* MCHist1D = MaCh3Sample->GetMCHist(SampleIndex, MaCh3Sample->GetNDim(SampleIndex)); - MC_Hist_Toy[SampleCounter][i] = M3::Clone(MCHist1D, MaCh3Sample->GetSampleTitle(SampleIndex) + "_mc_" + std::to_string(i)); + for (size_t iPDF = 0; iPDF < samples.size(); iPDF++) { + auto *MaCh3Sample = dynamic_cast(samples[iPDF]); + for (int SampleIndex = 0; SampleIndex < MaCh3Sample->GetNsamples(); + ++SampleIndex) { + TH1 *MCHist1D = MaCh3Sample->GetMCHist( + SampleIndex, MaCh3Sample->GetNDim(SampleIndex)); + MC_Hist_Toy[SampleCounter][i] = + M3::Clone(MCHist1D, MaCh3Sample->GetSampleTitle(SampleIndex) + + "_mc_" + std::to_string(i)); MC_Hist_Toy[SampleCounter][i]->Write(); - TH1* W2Hist1D = MaCh3Sample->GetW2Hist(SampleIndex, MaCh3Sample->GetNDim(SampleIndex)); - W2_Hist_Toy[SampleCounter][i] = M3::Clone(W2Hist1D, MaCh3Sample->GetSampleTitle(SampleIndex) + "_w2_" + std::to_string(i)); + TH1 *W2Hist1D = MaCh3Sample->GetW2Hist( + SampleIndex, MaCh3Sample->GetNDim(SampleIndex)); + W2_Hist_Toy[SampleCounter][i] = + M3::Clone(W2Hist1D, MaCh3Sample->GetSampleTitle(SampleIndex) + + "_w2_" + std::to_string(i)); W2_Hist_Toy[SampleCounter][i]->Write(); delete W2Hist1D; SampleCounter++; @@ -466,10 +518,11 @@ void PredictiveThrower::ProduceToys() { } ToyTree->Fill(); - }//end of toys loop + } // end of toys loop TempClock.Stop(); - if(PosteriorFile) delete PosteriorFile; + if (PosteriorFile) + delete PosteriorFile; ToyDirectory->Close(); delete ToyDirectory; @@ -477,16 +530,20 @@ void PredictiveThrower::ProduceToys() { ToyTree->Write(); delete ToyTree; - MACH3LOG_INFO("{} took {:.2f}s to finish for {} toys", __func__, TempClock.RealTime(), Ntoys); + MACH3LOG_INFO("{} took {:.2f}s to finish for {} toys", __func__, + TempClock.RealTime(), Ntoys); } // ************************* -std::vector>> PredictiveThrower::ProduceSpectra(const std::vector>>& Toys, - const std::string suffix) { -// ************************* +std::vector>> +PredictiveThrower::ProduceSpectra( + const std::vector>> &Toys, + const std::string suffix) { + // ************************* std::vector> MaxValue(TotalNumberOfSamples); - //Projections[sample][toy][dim] - std::vector>>> Projections(TotalNumberOfSamples); + // Projections[sample][toy][dim] + std::vector>>> Projections( + TotalNumberOfSamples); for (int sample = 0; sample < TotalNumberOfSamples; ++sample) { const int nDims = Toys[sample][0]->GetDimension(); @@ -498,7 +555,7 @@ std::vector>> PredictiveThrower::ProduceSpectr // Leave Projections[sample][toy][0] == nullptr } else if (nDims == 2) { Projections[sample][toy].resize(nDims); - TH2* h2 = dynamic_cast(Toys[sample][toy].get()); + TH2 *h2 = dynamic_cast(Toys[sample][toy].get()); if (!h2) { MACH3LOG_ERROR("Histogram is not TH2 for nDims==2"); throw MaCh3Exception(__FILE__, __LINE__); @@ -514,15 +571,17 @@ std::vector>> PredictiveThrower::ProduceSpectr delete px; delete py; } else { - MACH3LOG_ERROR("Asking for {} with N Dimension = {}. This is not implemented", __func__, nDims); + MACH3LOG_ERROR( + "Asking for {} with N Dimension = {}. This is not implemented", + __func__, nDims); throw MaCh3Exception(__FILE__, __LINE__); } } } - #ifdef MULTITHREAD - #pragma omp parallel for collapse(2) - #endif +#ifdef MULTITHREAD +#pragma omp parallel for collapse(2) +#endif for (int sample = 0; sample < TotalNumberOfSamples; ++sample) { for (int toy = 0; toy < Ntoys; ++toy) { const int nDims = Toys[sample][0]->GetDimension(); @@ -530,15 +589,16 @@ std::vector>> PredictiveThrower::ProduceSpectr double max_val = 0; if (nDims == 1) { max_val = Toys[sample][toy]->GetMaximum(); - } - else if (nDims == 2) { + } else if (nDims == 2) { if (dim == 0) { max_val = Projections[sample][toy][0]->GetMaximum(); } else { max_val = Projections[sample][toy][1]->GetMaximum(); } } else { - MACH3LOG_ERROR("Asking for {} with N Dimension = {}. This is not implemented", __func__, nDims); + MACH3LOG_ERROR( + "Asking for {} with N Dimension = {}. This is not implemented", + __func__, nDims); throw MaCh3Exception(__FILE__, __LINE__); } MaxValue[sample][dim] = std::max(MaxValue[sample][dim], max_val); @@ -552,31 +612,33 @@ std::vector>> PredictiveThrower::ProduceSpectr Spectra[sample].resize(nDims); for (int dim = 0; dim < nDims; dim++) { // Get MC histogram x-axis binning - TH1D* refHist = nullptr; + TH1D *refHist = nullptr; if (nDims == 1) { - refHist = static_cast(Toys[sample][0].get()); - } - else if (nDims == 2) { + refHist = static_cast(Toys[sample][0].get()); + } else if (nDims == 2) { if (dim == 0) { - refHist = static_cast(Projections[sample][0][0].get()); + refHist = static_cast(Projections[sample][0][0].get()); } else { - refHist = static_cast(Projections[sample][0][1].get()); + refHist = static_cast(Projections[sample][0][1].get()); } - } - else { - MACH3LOG_ERROR("Asking for {} with N Dimension = {}. This is not implemented", __func__, nDims); + } else { + MACH3LOG_ERROR( + "Asking for {} with N Dimension = {}. This is not implemented", + __func__, nDims); throw MaCh3Exception(__FILE__, __LINE__); } if (!refHist) { - MACH3LOG_ERROR("Failed to cast to {} dimensions in {}!", nDims, __func__); + MACH3LOG_ERROR("Failed to cast to {} dimensions in {}!", nDims, + __func__); throw MaCh3Exception(__FILE__, __LINE__); } const int n_bins_x = refHist->GetNbinsX(); std::vector x_bin_edges(n_bins_x + 1); for (int b = 0; b <= n_bins_x; ++b) { - x_bin_edges[b] = refHist->GetXaxis()->GetBinLowEdge(b + 1); // ROOT bins start at 1 + x_bin_edges[b] = + refHist->GetXaxis()->GetBinLowEdge(b + 1); // ROOT bins start at 1 } // Last edge is upper edge of last bin: x_bin_edges[n_bins_x] = refHist->GetXaxis()->GetBinUpEdge(n_bins_x); @@ -587,13 +649,16 @@ std::vector>> PredictiveThrower::ProduceSpectr // Create TH2D with variable binning on x axis Spectra[sample][dim] = std::make_unique( - (SampleNames[sample] + "_" + suffix + "_dim" + std::to_string(dim)).c_str(), // name - (SampleNames[sample] + "_" + suffix + "_dim" + std::to_string(dim)).c_str(), // title - n_bins_x, x_bin_edges.data(), // x axis bins - n_bins_y, y_min, y_max // y axis bins + (SampleNames[sample] + "_" + suffix + "_dim" + std::to_string(dim)) + .c_str(), // name + (SampleNames[sample] + "_" + suffix + "_dim" + std::to_string(dim)) + .c_str(), // title + n_bins_x, x_bin_edges.data(), // x axis bins + n_bins_y, y_min, y_max // y axis bins ); - Spectra[sample][dim]->GetXaxis()->SetTitle(refHist->GetXaxis()->GetTitle()); + Spectra[sample][dim]->GetXaxis()->SetTitle( + refHist->GetXaxis()->GetTitle()); Spectra[sample][dim]->GetYaxis()->SetTitle("Events"); Spectra[sample][dim]->SetDirectory(nullptr); @@ -601,21 +666,24 @@ std::vector>> PredictiveThrower::ProduceSpectr } } - #ifdef MULTITHREAD - #pragma omp parallel for - #endif +#ifdef MULTITHREAD +#pragma omp parallel for +#endif for (int sample = 0; sample < TotalNumberOfSamples; ++sample) { const int nDims = Toys[sample][0]->GetDimension(); for (int dim = 0; dim < nDims; dim++) { for (int toy = 0; toy < Ntoys; ++toy) { if (nDims == 1) { - FastViolinFill(Spectra[sample][dim].get(), static_cast(Toys[sample][toy].get())); - } - else if (nDims == 2) { - FastViolinFill(Spectra[sample][dim].get(), static_cast(Projections[sample][toy][dim].get())); - } - else { - MACH3LOG_ERROR("Asking for {} with N Dimension = {}. This is not implemented", __func__, nDims); + FastViolinFill(Spectra[sample][dim].get(), + static_cast(Toys[sample][toy].get())); + } else if (nDims == 2) { + FastViolinFill( + Spectra[sample][dim].get(), + static_cast(Projections[sample][toy][dim].get())); + } else { + MACH3LOG_ERROR( + "Asking for {} with N Dimension = {}. This is not implemented", + __func__, nDims); throw MaCh3Exception(__FILE__, __LINE__); } } @@ -626,73 +694,75 @@ std::vector>> PredictiveThrower::ProduceSpectr } // ************************* -std::unique_ptr PredictiveThrower::MakePredictive(const std::vector>& Toys, - const std::string& Sample_Name, - const std::string& suffix, - const bool DebugHistograms) { -// ************************* +std::unique_ptr +PredictiveThrower::MakePredictive(const std::vector> &Toys, + const std::string &Sample_Name, + const std::string &suffix, + const bool DebugHistograms) { + // ************************* int nDims = Toys[0]->GetDimension(); if (nDims == 1) { - int nbinsx = Toys[0]->GetNbinsX(); - const double* xbins = Toys[0]->GetXaxis()->GetXbins()->GetArray(); - auto PredictiveHist = std::make_unique( - (Sample_Name + "_" + suffix + "_PostPred").c_str(), - (Sample_Name + "_" + suffix + "_PostPred").c_str(), - nbinsx, xbins - ); - PredictiveHist->GetXaxis()->SetTitle(Toys[0]->GetXaxis()->GetTitle()); - PredictiveHist->GetYaxis()->SetTitle("Events"); - PredictiveHist->SetDirectory(nullptr); + int nbinsx = Toys[0]->GetNbinsX(); + const double *xbins = Toys[0]->GetXaxis()->GetXbins()->GetArray(); + auto PredictiveHist = std::make_unique( + (Sample_Name + "_" + suffix + "_PostPred").c_str(), + (Sample_Name + "_" + suffix + "_PostPred").c_str(), nbinsx, xbins); + PredictiveHist->GetXaxis()->SetTitle(Toys[0]->GetXaxis()->GetTitle()); + PredictiveHist->GetYaxis()->SetTitle("Events"); + PredictiveHist->SetDirectory(nullptr); - for (int i = 1; i <= nbinsx; ++i) { - TString projName = TString::Format("%s %s X Bin %d", Sample_Name.c_str(), suffix.c_str(), i); - auto PosteriorHist = std::make_unique(projName, projName, 100, 1, -1); - PosteriorHist->SetDirectory(nullptr); + for (int i = 1; i <= nbinsx; ++i) { + TString projName = TString::Format("%s %s X Bin %d", Sample_Name.c_str(), + suffix.c_str(), i); + auto PosteriorHist = + std::make_unique(projName, projName, 100, 1, -1); + PosteriorHist->SetDirectory(nullptr); - for (size_t iToy = 0; iToy < Toys.size(); ++iToy) { - double content = Toys[iToy]->GetBinContent(i); - PosteriorHist->Fill(content, ReweightWeight[iToy]); - } + for (size_t iToy = 0; iToy < Toys.size(); ++iToy) { + double content = Toys[iToy]->GetBinContent(i); + PosteriorHist->Fill(content, ReweightWeight[iToy]); + } - if (DebugHistograms) PosteriorHist->Write(); + if (DebugHistograms) + PosteriorHist->Write(); - PredictiveHist->SetBinContent(i, PosteriorHist->GetMean()); - PredictiveHist->SetBinError(i, PosteriorHist->GetRMS()); - } - PredictiveHist->Write(); - return PredictiveHist; - } - else if (nDims == 2) { + PredictiveHist->SetBinContent(i, PosteriorHist->GetMean()); + PredictiveHist->SetBinError(i, PosteriorHist->GetRMS()); + } + PredictiveHist->Write(); + return PredictiveHist; + } else if (nDims == 2) { int nbinsx = Toys[0]->GetNbinsX(); int nbinsy = Toys[0]->GetNbinsY(); - const double* xbins = Toys[0]->GetXaxis()->GetXbins()->GetArray(); - const double* ybins = Toys[0]->GetYaxis()->GetXbins()->GetArray(); + const double *xbins = Toys[0]->GetXaxis()->GetXbins()->GetArray(); + const double *ybins = Toys[0]->GetYaxis()->GetXbins()->GetArray(); // Create 2D predictive histogram with same binning as Toys[0] auto PredictiveHist = std::make_unique( (Sample_Name + "_" + suffix + "_PostPred").c_str(), - (Sample_Name + "_" + suffix + "_PostPred").c_str(), - nbinsx, xbins, - nbinsy, ybins - ); + (Sample_Name + "_" + suffix + "_PostPred").c_str(), nbinsx, xbins, + nbinsy, ybins); PredictiveHist->GetXaxis()->SetTitle(Toys[0]->GetXaxis()->GetTitle()); PredictiveHist->GetYaxis()->SetTitle(Toys[0]->GetYaxis()->GetTitle()); PredictiveHist->SetDirectory(nullptr); for (int ix = 1; ix <= nbinsx; ++ix) { for (int iy = 1; iy <= nbinsy; ++iy) { - TString projName = TString::Format("%s %s Bin (%d,%d)", Sample_Name.c_str(), suffix.c_str(), ix, iy); - auto PosteriorHist = std::make_unique(projName, projName, 100, 1, -1); + TString projName = TString::Format( + "%s %s Bin (%d,%d)", Sample_Name.c_str(), suffix.c_str(), ix, iy); + auto PosteriorHist = + std::make_unique(projName, projName, 100, 1, -1); PosteriorHist->SetDirectory(nullptr); PosteriorHist->GetXaxis()->SetTitle("Events"); int bin = Toys[0]->GetBin(ix, iy); for (size_t iToy = 0; iToy < Toys.size(); ++iToy) { - double content = Toys[iToy]->GetBinContent(bin); - PosteriorHist->Fill(content, ReweightWeight[iToy]); + double content = Toys[iToy]->GetBinContent(bin); + PosteriorHist->Fill(content, ReweightWeight[iToy]); } - if (DebugHistograms) PosteriorHist->Write(); + if (DebugHistograms) + PosteriorHist->Write(); PredictiveHist->SetBinContent(ix, iy, PosteriorHist->GetMean()); PredictiveHist->SetBinError(ix, iy, PosteriorHist->GetRMS()); @@ -700,51 +770,59 @@ std::unique_ptr PredictiveThrower::MakePredictive(const std::vectorWrite(); return PredictiveHist; - } - else { - MACH3LOG_ERROR("Asking for {} with N Dimension = {}. This is not implemented", __func__, nDims); - throw MaCh3Exception(__FILE__, __LINE__); + } else { + MACH3LOG_ERROR( + "Asking for {} with N Dimension = {}. This is not implemented", + __func__, nDims); + throw MaCh3Exception(__FILE__, __LINE__); } } // ************************* // Perform predictive analysis void PredictiveThrower::RunPredictiveAnalysis() { -// ************************* + // ************************* MACH3LOG_INFO("Starting {}", __func__); TStopwatch TempClock; TempClock.Start(); - auto DebugHistograms = GetFromManager(fitMan->raw()["Predictive"]["DebugHistograms"], false, __FILE__, __LINE__); + auto DebugHistograms = + GetFromManager(fitMan->raw()["Predictive"]["DebugHistograms"], + false, __FILE__, __LINE__); - TDirectory* PredictiveDir = outputFile->mkdir("Predictive"); - std::vector SampleDirectories; - SampleDirectories.resize(TotalNumberOfSamples+1); + TDirectory *PredictiveDir = outputFile->mkdir("Predictive"); + std::vector SampleDirectories; + SampleDirectories.resize(TotalNumberOfSamples + 1); - for (int sample = 0; sample < TotalNumberOfSamples+1; ++sample) { - SampleDirectories[sample] = PredictiveDir->mkdir(SampleNames[sample].c_str()); + for (int sample = 0; sample < TotalNumberOfSamples + 1; ++sample) { + SampleDirectories[sample] = + PredictiveDir->mkdir(SampleNames[sample].c_str()); } - std::vector>> Spectra_mc = ProduceSpectra(MC_Hist_Toy, "mc"); - //std::vector> Spectra_w2 = ProduceSpectra(W2_Hist_Toy, "w2"); + std::vector>> Spectra_mc = + ProduceSpectra(MC_Hist_Toy, "mc"); + // std::vector> Spectra_w2 = + // ProduceSpectra(W2_Hist_Toy, "w2"); std::vector> PostPred_mc(TotalNumberOfSamples); - //std::vector> PostPred_w2(TotalNumberOfSamples); + // std::vector> PostPred_w2(TotalNumberOfSamples); for (int sample = 0; sample < TotalNumberOfSamples; ++sample) { SampleDirectories[sample]->cd(); for (long unsigned int dim = 0; dim < Spectra_mc[sample].size(); dim++) { Spectra_mc[sample][dim]->Write(); } - //Spectra_w2[sample]->Write(); + // Spectra_w2[sample]->Write(); - PostPred_mc[sample] = MakePredictive(MC_Hist_Toy[sample], SampleNames[sample], "mc", DebugHistograms); - //PostPred_w2[sample] = MakePredictive(W2_Hist_Toy[sample], SampleNames[sample], "w2", DebugHistograms); + PostPred_mc[sample] = MakePredictive( + MC_Hist_Toy[sample], SampleNames[sample], "mc", DebugHistograms); + // PostPred_w2[sample] = MakePredictive(W2_Hist_Toy[sample], + // SampleNames[sample], "w2", DebugHistograms); } PosteriorPredictivepValue(PostPred_mc, - //PostPred_w2, + // PostPred_w2, SampleDirectories); // Close directories - for (int sample = 0; sample < TotalNumberOfSamples+1; ++sample) { + for (int sample = 0; sample < TotalNumberOfSamples + 1; ++sample) { SampleDirectories[sample]->Close(); delete SampleDirectories[sample]; } @@ -755,42 +833,43 @@ void PredictiveThrower::RunPredictiveAnalysis() { outputFile->cd(); TempClock.Stop(); - MACH3LOG_INFO("{} took {:.2f}s to finish for {} toys", __func__, TempClock.RealTime(), Ntoys); + MACH3LOG_INFO("{} took {:.2f}s to finish for {} toys", __func__, + TempClock.RealTime(), Ntoys); } // ************************* -double PredictiveThrower::GetLLH(const std::unique_ptr& DatHist, - const std::unique_ptr& MCHist, - const std::unique_ptr& W2Hist, - SampleHandlerBase* SampleHandler) { -// ************************* +double PredictiveThrower::GetLLH(const std::unique_ptr &DatHist, + const std::unique_ptr &MCHist, + const std::unique_ptr &W2Hist, + SampleHandlerBase *SampleHandler) { + // ************************* double llh = 0.0; - for (int i = 1; i <= DatHist->GetXaxis()->GetNbins(); ++i) - { + for (int i = 1; i <= DatHist->GetXaxis()->GetNbins(); ++i) { const double data = DatHist->GetBinContent(i); const double mc = MCHist->GetBinContent(i); const double w2 = W2Hist->GetBinContent(i); llh += SampleHandler->GetTestStatLLH(data, mc, w2); } - //KS: do times 2 because banff reports chi2 - return 2*llh; + // KS: do times 2 because banff reports chi2 + return 2 * llh; } // ************************* -void PredictiveThrower::PosteriorPredictivepValue(const std::vector>& PostPred_mc, - //const std::vector>& PostPred_w2, - const std::vector& SampleDir) { -// ************************* +void PredictiveThrower::PosteriorPredictivepValue( + const std::vector> &PostPred_mc, + // const std::vector>& PostPred_w2, + const std::vector &SampleDir) { + // ************************* //(void) PostPred_w2; // [Toys][Sample] std::vector> chi2_dat_vec(Ntoys); std::vector> chi2_mc_vec(Ntoys); std::vector> chi2_pred_vec(Ntoys); - for(int iToy = 0; iToy < Ntoys; iToy++) { - chi2_dat_vec[iToy].resize(TotalNumberOfSamples+1, 0); - chi2_mc_vec[iToy].resize(TotalNumberOfSamples+1, 0); - chi2_pred_vec[iToy].resize(TotalNumberOfSamples+1, 0); + for (int iToy = 0; iToy < Ntoys; iToy++) { + chi2_dat_vec[iToy].resize(TotalNumberOfSamples + 1, 0); + chi2_mc_vec[iToy].resize(TotalNumberOfSamples + 1, 0); + chi2_pred_vec[iToy].resize(TotalNumberOfSamples + 1, 0); chi2_dat_vec[iToy].back() = PenaltyTerm[iToy]; chi2_mc_vec[iToy].back() = PenaltyTerm[iToy]; @@ -803,42 +882,60 @@ void PredictiveThrower::PosteriorPredictivepValue(const std::vector(DrawFluctHist.get()), static_cast(MC_Hist_Toy[iSample][iToy].get()), random.get()); - MakeFluctuatedHistogramAlternative(static_cast(PredFluctHist.get()), static_cast(PostPred_mc[iSample].get()), random.get()); - } - else if (nDims == 2) { - MakeFluctuatedHistogramAlternative(static_cast(DrawFluctHist.get()), static_cast(MC_Hist_Toy[iSample][iToy].get()), random.get()); - MakeFluctuatedHistogramAlternative(static_cast(PredFluctHist.get()), static_cast(PostPred_mc[iSample].get()), random.get()); - } - else { - MACH3LOG_ERROR("Asking for {} with N Dimension = {}. This is not implemented", __func__, nDims); + MakeFluctuatedHistogramAlternative( + static_cast(DrawFluctHist.get()), + static_cast(MC_Hist_Toy[iSample][iToy].get()), + random.get()); + MakeFluctuatedHistogramAlternative( + static_cast(PredFluctHist.get()), + static_cast(PostPred_mc[iSample].get()), random.get()); + } else if (nDims == 2) { + MakeFluctuatedHistogramAlternative( + static_cast(DrawFluctHist.get()), + static_cast(MC_Hist_Toy[iSample][iToy].get()), + random.get()); + MakeFluctuatedHistogramAlternative( + static_cast(PredFluctHist.get()), + static_cast(PostPred_mc[iSample].get()), random.get()); + } else { + MACH3LOG_ERROR( + "Asking for {} with N Dimension = {}. This is not implemented", + __func__, nDims); throw MaCh3Exception(__FILE__, __LINE__); } // Okay now we can do our chi2 calculation for our sample - chi2_dat_vec[iToy][iSample] = GetLLH(Data_Hist[iSample], MC_Hist_Toy[iSample][iToy], W2_Hist_Toy[iSample][iToy], samples[SampleObjectMap[iSample]]); - chi2_mc_vec[iToy][iSample] = GetLLH(DrawFluctHist, MC_Hist_Toy[iSample][iToy], W2_Hist_Toy[iSample][iToy], samples[SampleObjectMap[iSample]]); - chi2_pred_vec[iToy][iSample] = GetLLH(PredFluctHist, MC_Hist_Toy[iSample][iToy], W2_Hist_Toy[iSample][iToy], samples[SampleObjectMap[iSample]]); - - chi2_dat_vec[iToy].back() += chi2_dat_vec[iToy][iSample]; - chi2_mc_vec[iToy].back() += chi2_mc_vec[iToy][iSample]; + chi2_dat_vec[iToy][iSample] = + GetLLH(Data_Hist[iSample], MC_Hist_Toy[iSample][iToy], + W2_Hist_Toy[iSample][iToy], samples[SampleObjectMap[iSample]]); + chi2_mc_vec[iToy][iSample] = + GetLLH(DrawFluctHist, MC_Hist_Toy[iSample][iToy], + W2_Hist_Toy[iSample][iToy], samples[SampleObjectMap[iSample]]); + chi2_pred_vec[iToy][iSample] = + GetLLH(PredFluctHist, MC_Hist_Toy[iSample][iToy], + W2_Hist_Toy[iSample][iToy], samples[SampleObjectMap[iSample]]); + + chi2_dat_vec[iToy].back() += chi2_dat_vec[iToy][iSample]; + chi2_mc_vec[iToy].back() += chi2_mc_vec[iToy][iSample]; chi2_pred_vec[iToy].back() += chi2_pred_vec[iToy][iSample]; } } - MakeChi2Plots(chi2_mc_vec, "-2LLH (Draw Fluc, Draw)", chi2_dat_vec, "-2LLH (Data, Draw)", SampleDir, "_drawfluc_draw"); - MakeChi2Plots(chi2_pred_vec, "-2LLH (Pred Fluc, Draw)", chi2_dat_vec, "-2LLH (Data, Draw)", SampleDir, "_predfluc_draw"); + MakeChi2Plots(chi2_mc_vec, "-2LLH (Draw Fluc, Draw)", chi2_dat_vec, + "-2LLH (Data, Draw)", SampleDir, "_drawfluc_draw"); + MakeChi2Plots(chi2_pred_vec, "-2LLH (Pred Fluc, Draw)", chi2_dat_vec, + "-2LLH (Data, Draw)", SampleDir, "_predfluc_draw"); } // ************************* -void PredictiveThrower::MakeChi2Plots(const std::vector>& Chi2_x, - const std::string& Chi2_x_title, - const std::vector>& Chi2_y, - const std::string& Chi2_y_title, - const std::vector& SampleDir, - const std::string Title) { -// ************************* - for (int iSample = 0; iSample < TotalNumberOfSamples+1; ++iSample) { +void PredictiveThrower::MakeChi2Plots( + const std::vector> &Chi2_x, + const std::string &Chi2_x_title, + const std::vector> &Chi2_y, + const std::string &Chi2_y_title, const std::vector &SampleDir, + const std::string Title) { + // ************************* + for (int iSample = 0; iSample < TotalNumberOfSamples + 1; ++iSample) { SampleDir[iSample]->cd(); // Transpose to extract chi2 values for a given sample across all toys @@ -847,17 +944,20 @@ void PredictiveThrower::MakeChi2Plots(const std::vector>& Ch for (int iToy = 0; iToy < Ntoys; ++iToy) { chi2_y_sample[iToy] = Chi2_y[iToy][iSample]; - chi2_x_per_sample[iToy] = Chi2_x[iToy][iSample]; + chi2_x_per_sample[iToy] = Chi2_x[iToy][iSample]; } - const double min_val = std::min(*std::min_element(chi2_y_sample.begin(), chi2_y_sample.end()), - *std::min_element(chi2_x_per_sample.begin(), chi2_x_per_sample.end())); - const double max_val = std::max(*std::max_element(chi2_y_sample.begin(), chi2_y_sample.end()), - *std::max_element(chi2_x_per_sample.begin(), chi2_x_per_sample.end())); + const double min_val = std::min( + *std::min_element(chi2_y_sample.begin(), chi2_y_sample.end()), + *std::min_element(chi2_x_per_sample.begin(), chi2_x_per_sample.end())); + const double max_val = std::max( + *std::max_element(chi2_y_sample.begin(), chi2_y_sample.end()), + *std::max_element(chi2_x_per_sample.begin(), chi2_x_per_sample.end())); - auto chi2_hist = std::make_unique((SampleNames[iSample] + Title).c_str(), - (SampleNames[iSample] + Title).c_str(), - 100, min_val, max_val, 100, min_val, max_val); + auto chi2_hist = + std::make_unique((SampleNames[iSample] + Title).c_str(), + (SampleNames[iSample] + Title).c_str(), 100, + min_val, max_val, 100, min_val, max_val); chi2_hist->SetDirectory(nullptr); chi2_hist->GetXaxis()->SetTitle(Chi2_x_title.c_str()); chi2_hist->GetYaxis()->SetTitle(Chi2_y_title.c_str()); diff --git a/Parameters/CMakeLists.txt b/Parameters/CMakeLists.txt index e15d1037f..eaf16dc94 100644 --- a/Parameters/CMakeLists.txt +++ b/Parameters/CMakeLists.txt @@ -10,7 +10,7 @@ set(HEADERS add_library(Parameters SHARED ParameterHandlerBase.cpp - # ParameterHandlerGeneric.cpp + ParameterHandlerGeneric.cpp ParameterTunes.cpp StepProposer.cpp ParameterList.cpp diff --git a/Parameters/ParameterHandlerBase.cpp b/Parameters/ParameterHandlerBase.cpp index f5a0128d3..0d97f8b61 100644 --- a/Parameters/ParameterHandlerBase.cpp +++ b/Parameters/ParameterHandlerBase.cpp @@ -502,6 +502,10 @@ bool ParameterHandlerBase::IsParameterFixed(const int) const { throw std::runtime_error("ParameterHandlerBase::IsParameterFixed Removed"); } +bool ParameterHandlerBase::IsPCParameterFixed(const int i) const { + return !proposer.params.isfree[i]; +} + // ******************************************** bool ParameterHandlerBase::IsParameterFixed(const std::string &) const { // ******************************************** @@ -523,6 +527,49 @@ void ParameterHandlerBase::SetFlatPrior(const int i, const bool eL) { } } +std::string ParameterHandlerBase::GetPCParName(const int i) { + int propidx = parlist.SystematicParameterIndexToPCIndex(i); + if (propidx != ParameterList::ParameterInPCABlock) { + return parlist.params.name[propidx]; + } else { + return "PCA_" + std::to_string(i - parlist.pca.first_index); + } +} + +double ParameterHandlerBase::GetPCParInit(const int i) { + int propidx = parlist.SystematicParameterIndexToPCIndex(i); + if (propidx != ParameterList::ParameterInPCABlock) { + return parlist.params.prefit[propidx]; + } else { + return 0; + } +} + +double ParameterHandlerBase::GetPCDiagonalError(const int i) { + int propidx = parlist.SystematicParameterIndexToPCIndex(i); + if (propidx != ParameterList::ParameterInPCABlock) { + return GetDiagonalError(propidx); + } else { + return 1; + } +} +double ParameterHandlerBase::GetPCLowerBound(const int i) { + int propidx = parlist.SystematicParameterIndexToPCIndex(i); + if (propidx != ParameterList::ParameterInPCABlock) { + return GetLowerBound(propidx); + } else { + return -3; + } +} +double ParameterHandlerBase::GetPCUpperBound(const int i) { + int propidx = parlist.SystematicParameterIndexToPCIndex(i); + if (propidx != ParameterList::ParameterInPCABlock) { + return GetUpperBound(propidx); + } else { + return 3; + } +} + void ParameterHandlerBase::SetIndivStepScale(const int ParameterIndex, const double StepScale) { int idx = parlist.SystematicParameterIndexToPCIndex(ParameterIndex); @@ -607,29 +654,25 @@ void ParameterHandlerBase::SetThrowMatrix(TMatrixDSym *cov) { M3::MakeMatrixPosDef(cov); Eigen::MatrixXd covariance = M3::ROOTToEigen(*cov); - if (parlist.pca.enabled) { - Eigen::MatrixXd covariance_pc = Eigen::MatrixXd::Zero( - proposer.NumParameters(), proposer.NumParameters()); + Eigen::MatrixXd covariance_pc = + Eigen::MatrixXd::Zero(proposer.NumParameters(), proposer.NumParameters()); - covariance_pc.topLeftCorner(parlist.pca.first_index, - parlist.pca.first_index) = - parlist.params.covariance.topLeftCorner(parlist.pca.first_index, - parlist.pca.first_index); + covariance_pc.topLeftCorner(parlist.pca.first_index, + parlist.pca.first_index) = + parlist.params.covariance.topLeftCorner(parlist.pca.first_index, + parlist.pca.first_index); - covariance_pc.block(parlist.pca.first_index, parlist.pca.first_index, - parlist.pca.npc_parameters(), - parlist.pca.npc_parameters()) = - Eigen::MatrixXd::Identity(parlist.pca.npc_parameters(), - parlist.pca.npc_parameters()); + covariance_pc.block(parlist.pca.first_index, parlist.pca.first_index, + parlist.pca.npc_parameters(), + parlist.pca.npc_parameters()) = + Eigen::MatrixXd::Identity(parlist.pca.npc_parameters(), + parlist.pca.npc_parameters()); - covariance_pc.bottomRightCorner(parlist.pca.ntail, parlist.pca.ntail) = - parlist.params.covariance.bottomRightCorner(parlist.pca.ntail, - parlist.pca.ntail); + covariance_pc.bottomRightCorner(parlist.pca.ntail, parlist.pca.ntail) = + parlist.params.covariance.bottomRightCorner(parlist.pca.ntail, + parlist.pca.ntail); - proposer.SetProposalMatrix(covariance_pc); - } else { - proposer.SetProposalMatrix(covariance); - } + proposer.SetProposalMatrix(covariance_pc); } void ParameterHandlerBase::UpdateThrowMatrix(TMatrixDSym *) { @@ -645,8 +688,8 @@ TH2D *ParameterHandlerBase::GetCorrelationMatrix() { parlist.params.covariance); } -const double *ParameterHandlerBase::RetPointer(const int) { - throw std::runtime_error("ParameterHandlerBase::RetPointer Removed"); +const double *ParameterHandlerBase::RetPointer(const int i) { + return current.data() + i; } // ******************************************** @@ -753,10 +796,10 @@ void ParameterHandlerBase::MatchMaCh3OutputBranches( TTree *PosteriorFile, std::vector &BranchValues, std::vector &BranchNames) { // ************************************* - BranchValues.resize(GetNumParams()); - BranchNames.resize(GetNumParams()); + BranchValues.resize(GetNumSystematicParams()); + BranchNames.resize(GetNumSystematicParams()); - for (int i = 0; i < GetNumParams(); ++i) { + for (int i = 0; i < GetNumSystematicParams(); ++i) { BranchNames[i] = GetParName(i); if (!PosteriorFile->GetBranch(BranchNames[i].c_str())) { MACH3LOG_ERROR("Branch '{}' does not exist in the TTree!", diff --git a/Parameters/ParameterHandlerBase.h b/Parameters/ParameterHandlerBase.h index 5ccbb2100..f46201e90 100644 --- a/Parameters/ParameterHandlerBase.h +++ b/Parameters/ParameterHandlerBase.h @@ -202,6 +202,12 @@ class ParameterHandlerBase { std::string GetParName(const int i) const { return parlist.params.name.at(i); } + //need these 'reach into PC' functions for MINUIT + std::string GetPCParName(const int i); + double GetPCParInit(const int i); + double GetPCDiagonalError(const int i); + double GetPCLowerBound(const int i); + double GetPCUpperBound(const int i); /// @brief Get index based on name /// @ingroup ParameterHandlerGetters @@ -293,7 +299,10 @@ class ParameterHandlerBase { /// @brief Get total number of parameters /// @ingroup ParameterHandlerGetters - int GetNumParams() const { return parlist.NumSystematicBasisParameters(); } + int GetNumSystematicParams() const { + return parlist.NumSystematicBasisParameters(); + } + int GetNumProposalParams() const { return parlist.NumPCBasisParameters(); } /// @brief Get the pre-fit values of the parameters. /// @ingroup ParameterHandlerGetters std::vector GetPreFitValues() const { @@ -397,6 +406,8 @@ class ParameterHandlerBase { /// @brief Is parameter fixed or not /// @param i Parameter index bool IsParameterFixed(const int i) const; + bool IsPCParameterFixed(const int i) const; + /// @brief Is parameter fixed or not /// @param name Name of parameter you want to check if is fixed bool IsParameterFixed(const std::string &name) const; @@ -412,9 +423,6 @@ class ParameterHandlerBase { void ConstructPCA(const double eigen_threshold, int FirstPCAdpar, int LastPCAdpar); - /// @brief is PCA, can use to query e.g. LLH scans - bool IsPCA() const { return parlist.pca.enabled; } - /// @brief Getter to return a copy of the YAML node /// @ingroup ParameterHandlerGetters YAML::Node _fYAMLDoc; @@ -445,9 +453,10 @@ class ParameterHandlerBase { bool AppliesToSample(const int SystIndex, const std::string &SampleName) const; - ParameterList parlist; - StepProposer proposer; - mutable Eigen::ArrayXd current; mutable Eigen::ArrayXd proposed; + +public: + ParameterList parlist; + StepProposer proposer; }; From 2692e667e8f080061dba8ce130c4c3a436d66166 Mon Sep 17 00:00:00 2001 From: Luke Pickering Date: Thu, 18 Dec 2025 17:00:35 +0000 Subject: [PATCH 10/14] cmake lists tweaks --- Parameters/CMakeLists.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Parameters/CMakeLists.txt b/Parameters/CMakeLists.txt index eaf16dc94..64b339e2f 100644 --- a/Parameters/CMakeLists.txt +++ b/Parameters/CMakeLists.txt @@ -2,10 +2,11 @@ set(HEADERS ParameterHandlerUtils.h ParameterHandlerBase.h ParameterHandlerGeneric.h - AdaptiveMCMCHandler.h - PCAHandler.h + PCA.h ParameterTunes.h ParameterStructs.h + StepProposer.h + ParameterList.h ) add_library(Parameters SHARED From 62d5754b14602ceaa5abed028299233f9f367d3a Mon Sep 17 00:00:00 2001 From: Luke Pickering Date: Thu, 18 Dec 2025 17:16:15 +0000 Subject: [PATCH 11/14] got overzealous in rebase and removed a header --- Fitters/PredictiveThrower.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Fitters/PredictiveThrower.cpp b/Fitters/PredictiveThrower.cpp index d33d37429..4807eb8d5 100644 --- a/Fitters/PredictiveThrower.cpp +++ b/Fitters/PredictiveThrower.cpp @@ -1,5 +1,7 @@ #include "PredictiveThrower.h" #include "Parameters/ParameterHandlerGeneric.h" +#include "Samples/SampleHandlerFD.h" + #include "TH3.h" // ************************* @@ -351,7 +353,7 @@ void PredictiveThrower::ProduceToys() { std::vector ParampPointers(NModelParams); int ParamCounter = 0; for (size_t iSys = 0; iSys < systematics.size(); iSys++) { - for (int iPar = 0; iPar < systematics[iSys]->GetNumParams(); iPar++) { + for (int iPar = 0; iPar < systematics[iSys]->GetNumSystematicParams(); iPar++) { ParampPointers[ParamCounter] = systematics[iSys]->RetPointer(iPar); std::string Name = systematics[iSys]->GetParName(iPar); ToyTree->Branch(Name.c_str(), &ParamValues[ParamCounter], From 67360a28cc9371a2a98b01e89c49768eae0875b3 Mon Sep 17 00:00:00 2001 From: Luke Pickering Date: Fri, 19 Dec 2025 10:45:31 +0000 Subject: [PATCH 12/14] move basis translation to the step proposer - parameter list now just knows everything about the parameters - on proposal function creation, truncated PCA parameters can be passed - Using this abstraction seems more ergonomic as you don't need to keep going back to the parameter list to get the 'global' systematic parameter values - tests disabled for the moment as I'm in a bit of a rush to build mach3 dune --- Fitters/LikelihoodFit.cpp | 2 +- Fitters/MinuitFit.cpp | 22 +-- Fitters/PSO.cpp | 10 +- Parameters/CMakeLists.txt | 2 +- Parameters/ParameterHandlerBase.cpp | 243 ++++++++++++------------- Parameters/ParameterHandlerBase.h | 28 +-- Parameters/ParameterHandlerGeneric.cpp | 47 ++--- Parameters/ParameterList.cpp | 198 ++++++++------------ Parameters/ParameterList.h | 37 +--- Parameters/README.md | 3 +- Parameters/StepProposer.cpp | 141 +++++++++++--- Parameters/StepProposer.h | 58 +++++- 12 files changed, 416 insertions(+), 375 deletions(-) diff --git a/Fitters/LikelihoodFit.cpp b/Fitters/LikelihoodFit.cpp index b0a64feac..5bef18e75 100644 --- a/Fitters/LikelihoodFit.cpp +++ b/Fitters/LikelihoodFit.cpp @@ -46,7 +46,7 @@ double LikelihoodFit::CalcChi2PC(const double *x) { // set the parameters in the sampler basis std::copy_n(x + ParCounter, parhandlr->GetNumProposalParams(), - parhandlr->proposer.params.proposed.data()); + parhandlr->proposer.proposal_basis.proposed.data()); ParCounter += parhandlr->GetNumProposalParams(); // this accepts the step for the proposer and rotates the parameters back to diff --git a/Fitters/MinuitFit.cpp b/Fitters/MinuitFit.cpp index 6f46f04a9..66cc92bd0 100644 --- a/Fitters/MinuitFit.cpp +++ b/Fitters/MinuitFit.cpp @@ -65,15 +65,15 @@ void MinuitFit::RunMCMC() { for (int i = 0; i < parhandlr->GetNumProposalParams(); ++i, ++ParCounter) { // KS: Index, name, prior, step scale [different to MCMC], - minuit->SetVariable(ParCounter, (parhandlr->GetPCParName(i)), - parhandlr->GetPCParInit(i), - parhandlr->GetPCDiagonalError(i) / 10.0); - minuit->SetVariableValue(ParCounter, parhandlr->GetPCParInit(i)); + minuit->SetVariable(ParCounter, (parhandlr->GetProposalParName(i)), + parhandlr->GetProposalParInit(i), + parhandlr->GetProposalDiagonalError(i) / 10.0); + minuit->SetVariableValue(ParCounter, parhandlr->GetProposalParInit(i)); // KS: lower bound, upper bound, if Mirroring enabled then ignore if (!fMirroring) - minuit->SetVariableLimits(ParCounter, parhandlr->GetPCLowerBound(i), - parhandlr->GetPCUpperBound(i)); - if (parhandlr->IsPCParameterFixed(i)) { + minuit->SetVariableLimits(ParCounter, parhandlr->GetProposalLowerBound(i), + parhandlr->GetProposalUpperBound(i)); + if (parhandlr->IsProposalParameterFixed(i)) { minuit->FixVariable(ParCounter); } } @@ -108,7 +108,7 @@ void MinuitFit::RunMCMC() { // set the parameters in the sampler basis std::copy_n(X + ParCounter, parhandlr->GetNumProposalParams(), - parhandlr->proposer.params.proposed.data()); + parhandlr->proposer.proposal_basis.proposed.data()); // this accepts the step for the proposer and rotates the parameters back to // the systematic basis @@ -133,13 +133,13 @@ void MinuitFit::RunMCMC() { for (int i = 0; i < parhandlr->GetNumProposalParams(); ++i) { (*MinuitParValue)(ParCounter + i) = - parhandlr->proposer.params.proposed[i]; + parhandlr->proposer.proposal_basis.proposed[i]; (*MinuitParError)(ParCounter + i) = err[ParCounter + i]; // KS: For fixed params HESS will not calculate error so we need to pass // prior error - if (parhandlr->IsPCParameterFixed(i)) { - (*MinuitParError)(ParCounter) = parhandlr->GetPCDiagonalError(i); + if (parhandlr->IsProposalParameterFixed(i)) { + (*MinuitParError)(ParCounter) = parhandlr->GetProposalDiagonalError(i); (*Postmatrix)(ParCounter, ParCounter) = (*MinuitParError)(ParCounter) * (*MinuitParError)(ParCounter); } diff --git a/Fitters/PSO.cpp b/Fitters/PSO.cpp index 8d6506705..597ab3a41 100644 --- a/Fitters/PSO.cpp +++ b/Fitters/PSO.cpp @@ -80,10 +80,10 @@ void PSO::init() { for (auto &parhandlr : systematics) { fDim += parhandlr->GetNumProposalParams(); for (int i = 0; i < parhandlr->GetNumProposalParams(); ++i) { - double curr = parhandlr->GetPCParInit(i); - double lim = 10.0 * parhandlr->GetPCDiagonalError(i); - double low = parhandlr->GetPCLowerBound(i); - double high = parhandlr->GetPCUpperBound(i); + double curr = parhandlr->GetProposalParInit(i); + double lim = 10.0 * parhandlr->GetProposalDiagonalError(i); + double low = parhandlr->GetProposalLowerBound(i); + double high = parhandlr->GetProposalUpperBound(i); if (low > curr - lim) ranges_min.push_back(low); else @@ -94,7 +94,7 @@ void PSO::init() { ranges_min.push_back(curr + lim); prior.push_back(curr); - if (parhandlr->IsPCParameterFixed(i)) { + if (parhandlr->IsProposalParameterFixed(i)) { fixed.push_back(1); } else { fixed.push_back(0); diff --git a/Parameters/CMakeLists.txt b/Parameters/CMakeLists.txt index 64b339e2f..21bb8387f 100644 --- a/Parameters/CMakeLists.txt +++ b/Parameters/CMakeLists.txt @@ -52,4 +52,4 @@ install(TARGETS Parameters add_library(MaCh3::Parameters ALIAS Parameters) -add_subdirectory(test) +# add_subdirectory(test) diff --git a/Parameters/ParameterHandlerBase.cpp b/Parameters/ParameterHandlerBase.cpp index 0d97f8b61..d55c55f05 100644 --- a/Parameters/ParameterHandlerBase.cpp +++ b/Parameters/ParameterHandlerBase.cpp @@ -22,9 +22,9 @@ ParameterHandlerBase::ParameterHandlerBase(std::string name, std::string file, proposer = parlist.MakeProposer(); - throwMatrix = new TMatrixDSym(int(proposer.params.proposal.rows()), - int(proposer.params.proposal.cols())); - M3::EigenToROOT(proposer.params.proposal, *throwMatrix); + throwMatrix = new TMatrixDSym(int(proposer.proposal_basis.proposal.rows()), + int(proposer.proposal_basis.proposal.cols())); + M3::EigenToROOT(proposer.proposal_basis.proposal, *throwMatrix); } // ******************************************** ParameterHandlerBase::ParameterHandlerBase( @@ -62,21 +62,14 @@ ParameterHandlerBase::ParameterHandlerBase( M3::EigenToROOT(parlist.params.inv_covariance, *invCovMatrix); if (pca) { - ConstructPCA(threshold, FirstPCA, LastPCA); + proposer = parlist.MakePCAProposer(threshold, FirstPCA, LastPCA); + } else { + proposer = parlist.MakeProposer(); } - proposer = parlist.MakeProposer(); - - throwMatrix = new TMatrixDSym(int(proposer.params.proposal.rows()), - int(proposer.params.proposal.cols())); - M3::EigenToROOT(proposer.params.proposal, *throwMatrix); -} - -// ******************************************** -void ParameterHandlerBase::ConstructPCA(const double threshold, - int FirstPCAdpar, int LastPCAdpar) { - // ******************************************** - parlist.ConstructTruncatedPCA(threshold, FirstPCAdpar, LastPCAdpar); + throwMatrix = new TMatrixDSym(int(proposer.proposal_basis.proposal.rows()), + int(proposer.proposal_basis.proposal.cols())); + M3::EigenToROOT(proposer.proposal_basis.proposal, *throwMatrix); } // ******************************************** @@ -99,16 +92,14 @@ void ParameterHandlerBase::SetPar(int i, double val) { MACH3LOG_INFO("_fPropVal ({}), _fCurrVal ({}), _fPreFitValue ({}) to ({})", proposed[i], current[i], parlist.params.prefit[i], val); - parlist.RotateSystematicParameterValuesToPCBasis(current, - proposer.params.current); - parlist.RotateSystematicParameterValuesToPCBasis(proposed, - proposer.params.proposed); + proposer.SetSystematicParameterValues(current); } // ******************************************** std::vector ParameterHandlerBase::GetProposed() const { // ******************************************** - return M3::EigenToStdVector(proposer.params.proposed); + proposer.GetSystematicParameterValues(proposed); + return M3::EigenToStdVector(proposed); } // ************************************* @@ -136,10 +127,8 @@ void ParameterHandlerBase::ThrowParameters() { proposer.Accept(); - parlist.RotatePCParameterValuesToSystematicBasis(proposer.params.current, - current); - parlist.RotatePCParameterValuesToSystematicBasis(proposer.params.proposed, - proposed); + proposer.GetSystematicParameterValues(current); + proposed = current; } // ************************************* @@ -152,7 +141,7 @@ void ParameterHandlerBase::RandomConfiguration() { // likelihood Want to change the _fPropVal, _fCurrVal, _fPreFitValue // _fPreFitValue and the others will already be set - for (int i = 0; i < parlist.NumSystematicBasisParameters(); ++i) { + for (int i = 0; i < parlist.NumParameters(); ++i) { // Check if parameter is fixed first: if so don't randomly throw if (IsParameterFixed(i)) { continue; @@ -188,8 +177,7 @@ void ParameterHandlerBase::RandomConfiguration() { MACH3LOG_INFO("Setting current step in {} param {}, {} = {} from {}", GetName(), i, GetParName(i), current[i], current[i]); } - parlist.RotateSystematicParameterValuesToPCBasis(current, - proposer.params.proposed); + proposer.SetSystematicParameterValues(current); proposer.Accept(); proposed = current; } @@ -211,8 +199,7 @@ void ParameterHandlerBase::SetParCurrProp(const int parNo, void ParameterHandlerBase::SetParProp(const int i, const double val) { proposed[i] = val; - parlist.RotateSystematicParameterValuesToPCBasis(proposed, - proposer.params.proposed); + proposer.SetSystematicParameterValues(proposed); } void ParameterHandlerBase::SetRandomThrow(const int, const double) { @@ -230,8 +217,7 @@ double ParameterHandlerBase::GetRandomThrow(const int) const { void ParameterHandlerBase::ProposeStep() { // ************************************************ proposer.Propose(); - parlist.RotatePCParameterValuesToSystematicBasis(proposer.params.proposed, - proposed); + proposer.GetSystematicParameterValues(proposed); } // ******************************************** @@ -239,8 +225,7 @@ void ParameterHandlerBase::ProposeStep() { void ParameterHandlerBase::AcceptStep() _noexcept_ { // ******************************************** proposer.Accept(); - parlist.RotatePCParameterValuesToSystematicBasis(proposer.params.current, - current); + current = proposed; } // ******************************************** @@ -249,11 +234,11 @@ void ParameterHandlerBase::AcceptStep() _noexcept_ { void ParameterHandlerBase::ThrowParProp(const double mag) { // ******************************************** Eigen::VectorXd new_proposed = - ((proposer.Propose() - proposer.params.current) * mag) + - proposer.params.current; + ((proposer.Propose() - proposer.proposal_basis.current) * mag) + + proposer.proposal_basis.current; // probably fine but avoid ailiasing for now - proposer.params.proposed = new_proposed; - parlist.RotatePCParameterValuesToSystematicBasis(new_proposed, proposed); + proposer.proposal_basis.proposed = new_proposed; + proposer.GetSystematicParameterValues(proposed); } // ******************************************** // Helper function to throw the current parameter by mag sigmas @@ -261,21 +246,21 @@ void ParameterHandlerBase::ThrowParProp(const double mag) { void ParameterHandlerBase::ThrowParCurr(const double mag) { // ******************************************** - Eigen::VectorXd curr_proposed = proposer.params.proposed; + Eigen::VectorXd curr_proposed = proposer.proposal_basis.proposed; Eigen::VectorXd new_curr = - ((proposer.Propose() - proposer.params.current) * mag); - proposer.params.current = new_curr; + ((proposer.Propose() - proposer.proposal_basis.current) * mag); + proposer.proposal_basis.current = new_curr; // probably fine but avoid ailiasing for now - proposer.params.proposed = curr_proposed; + proposer.proposal_basis.proposed = curr_proposed; - parlist.RotatePCParameterValuesToSystematicBasis(new_curr, current); + proposer.TranformProposalParametersToSystematicBasis(new_curr, current); } // ******************************************** // Function to print the prior values void ParameterHandlerBase::PrintNominal() const { // ******************************************** MACH3LOG_INFO("Prior values for {} ParameterHandler:", GetName()); - for (int i = 0; i < parlist.NumSystematicBasisParameters(); i++) { + for (int i = 0; i < parlist.NumParameters(); i++) { MACH3LOG_INFO(" {} {} ", GetParFancyName(i), GetParInit(i)); } } @@ -285,16 +270,16 @@ void ParameterHandlerBase::PrintNominal() const { void ParameterHandlerBase::PrintNominalCurrProp() const { // ******************************************** - parlist.RotatePCParameterValuesToSystematicBasis(proposer.params.current, - current); + proposer.TranformProposalParametersToSystematicBasis( + proposer.proposal_basis.current, current); - parlist.RotatePCParameterValuesToSystematicBasis(proposer.params.proposed, - proposed); + proposer.TranformProposalParametersToSystematicBasis( + proposer.proposal_basis.proposed, proposed); MACH3LOG_INFO("Printing parameters for {}", GetName()); MACH3LOG_INFO("{:<30} {:<10} {:<10} {:<10}", "Name", "Prior", "Current", "Proposed"); - for (int i = 0; i < parlist.NumSystematicBasisParameters(); ++i) { + for (int i = 0; i < parlist.NumParameters(); ++i) { MACH3LOG_INFO("{:<30} {:<10.2f} {:<10.2f} {:<10.2f}", GetParFancyName(i), parlist.params.prefit[i], proposed[i], current[i]); } @@ -308,8 +293,7 @@ void ParameterHandlerBase::PrintNominalCurrProp() const { // false = evaluate likelihood (so run with a prior) double ParameterHandlerBase::CalcLikelihood() _noexcept_ { // ******************************************** - parlist.RotatePCParameterValuesToSystematicBasis(proposer.params.proposed, - proposed); + proposer.GetSystematicParameterValues(proposed); return parlist.Chi2(proposed); } @@ -318,8 +302,7 @@ double ParameterHandlerBase::CalcLikelihood() _noexcept_ { int ParameterHandlerBase::CheckBounds() const _noexcept_ { // ******************************************** - parlist.RotatePCParameterValuesToSystematicBasis(proposer.params.proposed, - proposed); + proposer.GetSystematicParameterValues(proposed); return int(((proposed.array() < parlist.params.lowbound.array()) || (proposed.array() >= parlist.params.upbound.array())) @@ -349,8 +332,8 @@ double ParameterHandlerBase::GetCorrThrows(const int) const { } bool ParameterHandlerBase::GetFlatPrior(const int i) const { - int propidx = parlist.SystematicParameterIndexToPCIndex(i); - if (propidx != ParameterList::ParameterInPCABlock) { + int propidx = proposer.GetProposalParameterIndexFromSystematicIndex(i); + if (propidx != StepProposer::ParameterInPCABlock) { return parlist.params.flatprior[i]; } else { MACH3LOG_ERROR("You are trying Fix parameter {}, {}, which has been PCA'd.", @@ -366,19 +349,16 @@ void ParameterHandlerBase::SetParameters(const std::vector &pars) { // If empty, set the proposed to prior if (pars.empty()) { // For xsec this means setting to the prior (because prior is the prior) - parlist.RotateSystematicParameterValuesToPCBasis(parlist.params.prefit, - proposer.params.proposed); + proposer.SetSystematicParameterValues(parlist.params.prefit); // If not empty, set the parameters to the specified } else { - if (int(pars.size()) != parlist.NumSystematicBasisParameters()) { + if (int(pars.size()) != parlist.NumParameters()) { MACH3LOG_ERROR("Parameter arrays of incompatible size! Not changing " "parameters! {} has size {} but was expecting {}", - GetName(), pars.size(), - parlist.NumSystematicBasisParameters()); + GetName(), pars.size(), parlist.NumParameters()); throw MaCh3Exception(__FILE__, __LINE__); } - parlist.RotateSystematicParameterValuesToPCBasis(M3::StdVectorToEigen(pars), - proposer.params.proposed); + proposer.SetSystematicParameterValues(M3::StdVectorToEigen(pars)); } } @@ -386,13 +366,13 @@ void ParameterHandlerBase::SetParameters(const std::vector &pars) { void ParameterHandlerBase::SetBranches(TTree &tree, bool SaveProposal) { // ******************************************** // loop over parameters and set a branch - for (int i = 0; i < parlist.NumSystematicBasisParameters(); ++i) { + for (int i = 0; i < parlist.NumParameters(); ++i) { tree.Branch(GetParName(i).c_str(), ¤t.data()[i], Form("%s/D", GetParName(i).c_str())); } if (SaveProposal) { // loop over parameters and set a branch - for (int i = 0; i < parlist.NumSystematicBasisParameters(); ++i) { + for (int i = 0; i < parlist.NumParameters(); ++i) { tree.Branch(Form("%s_Prop", GetParName(i).c_str()), &proposed.data()[i], Form("%s_Prop/D", GetParName(i).c_str())); } @@ -411,15 +391,14 @@ void ParameterHandlerBase::SetStepScale(const double scale, if (verbose) { MACH3LOG_INFO("{} setStepScale() = {}", GetName(), scale); - const double SuggestedScale = - 2.38 / std::sqrt(parlist.NumPCBasisParameters()); + const double SuggestedScale = 2.38 / std::sqrt(parlist.NumParameters()); if (std::fabs(scale - SuggestedScale) / SuggestedScale > 1) { MACH3LOG_WARN( "Defined Global StepScale is {}, while suggested suggested {}", scale, SuggestedScale); } } - proposer.params.global_scale = scale; + proposer.proposal_basis.global_scale = scale; } // ******************************************** @@ -431,15 +410,17 @@ int ParameterHandlerBase::GetParIndex(const std::string &name) const { // ******************************************** void ParameterHandlerBase::SetFixAllParameters() { // ******************************************** - proposer.params.isfree = Eigen::ArrayXi::Ones(proposer.NumParameters()); + proposer.systematic_basis.isfree.setZero(); + proposer.proposal_basis.isfree.setZero(); } // ******************************************** void ParameterHandlerBase::SetFixParameter(const int i) { // ******************************************** - int propidx = parlist.SystematicParameterIndexToPCIndex(i); - if (propidx != ParameterList::ParameterInPCABlock) { - proposer.params.isfree[propidx] = false; + int propidx = proposer.GetProposalParameterIndexFromSystematicIndex(i); + if (propidx != StepProposer::ParameterInPCABlock) { + proposer.proposal_basis.isfree[propidx] = false; + proposer.systematic_basis.isfree[i] = false; } else { MACH3LOG_ERROR("You are trying Fix parameter {}, {}, which has been PCA'd.", i, GetParName(i)); @@ -456,15 +437,17 @@ void ParameterHandlerBase::SetFixParameter(const std::string &name) { // ******************************************** void ParameterHandlerBase::SetFreeAllParameters() { // ******************************************** - proposer.params.isfree = Eigen::ArrayXi::Ones(proposer.NumParameters()); + proposer.systematic_basis.isfree.setOnes(); + proposer.proposal_basis.isfree.setOnes(); } // ******************************************** void ParameterHandlerBase::SetFreeParameter(const int i) { // ******************************************** - int propidx = parlist.SystematicParameterIndexToPCIndex(i); - if (propidx != ParameterList::ParameterInPCABlock) { - proposer.params.isfree[propidx] = true; + int propidx = proposer.GetProposalParameterIndexFromSystematicIndex(i); + if (propidx != StepProposer::ParameterInPCABlock) { + proposer.proposal_basis.isfree[propidx] = true; + proposer.systematic_basis.isfree[i] = true; } else { MACH3LOG_ERROR( "You are trying Free parameter {}, {}, which has been PCA'd.", i, @@ -502,8 +485,8 @@ bool ParameterHandlerBase::IsParameterFixed(const int) const { throw std::runtime_error("ParameterHandlerBase::IsParameterFixed Removed"); } -bool ParameterHandlerBase::IsPCParameterFixed(const int i) const { - return !proposer.params.isfree[i]; +bool ParameterHandlerBase::IsProposalParameterFixed(const int i) const { + return !proposer.proposal_basis.isfree[i]; } // ******************************************** @@ -516,8 +499,8 @@ bool ParameterHandlerBase::IsParameterFixed(const std::string &) const { void ParameterHandlerBase::SetFlatPrior(const int i, const bool eL) { // ******************************************** - int propidx = parlist.SystematicParameterIndexToPCIndex(i); - if (propidx != ParameterList::ParameterInPCABlock) { + int propidx = proposer.GetProposalParameterIndexFromSystematicIndex(i); + if (propidx != StepProposer::ParameterInPCABlock) { parlist.params.flatprior[i] = eL; } else { MACH3LOG_ERROR("You are trying set a flat prior for parameter {}, {}, " @@ -527,43 +510,44 @@ void ParameterHandlerBase::SetFlatPrior(const int i, const bool eL) { } } -std::string ParameterHandlerBase::GetPCParName(const int i) { - int propidx = parlist.SystematicParameterIndexToPCIndex(i); - if (propidx != ParameterList::ParameterInPCABlock) { +std::string ParameterHandlerBase::GetProposalParName(const int i) { + int propidx = proposer.GetProposalParameterIndexFromSystematicIndex(i); + if (propidx != StepProposer::ParameterInPCABlock) { return parlist.params.name[propidx]; } else { - return "PCA_" + std::to_string(i - parlist.pca.first_index); + return "PCA_" + + std::to_string(i - proposer.systematic_basis.pca.first_index); } } -double ParameterHandlerBase::GetPCParInit(const int i) { - int propidx = parlist.SystematicParameterIndexToPCIndex(i); - if (propidx != ParameterList::ParameterInPCABlock) { +double ParameterHandlerBase::GetProposalParInit(const int i) { + int propidx = proposer.GetProposalParameterIndexFromSystematicIndex(i); + if (propidx != StepProposer::ParameterInPCABlock) { return parlist.params.prefit[propidx]; } else { return 0; } } -double ParameterHandlerBase::GetPCDiagonalError(const int i) { - int propidx = parlist.SystematicParameterIndexToPCIndex(i); - if (propidx != ParameterList::ParameterInPCABlock) { +double ParameterHandlerBase::GetProposalDiagonalError(const int i) { + int propidx = proposer.GetProposalParameterIndexFromSystematicIndex(i); + if (propidx != StepProposer::ParameterInPCABlock) { return GetDiagonalError(propidx); } else { return 1; } } -double ParameterHandlerBase::GetPCLowerBound(const int i) { - int propidx = parlist.SystematicParameterIndexToPCIndex(i); - if (propidx != ParameterList::ParameterInPCABlock) { +double ParameterHandlerBase::GetProposalLowerBound(const int i) { + int propidx = proposer.GetProposalParameterIndexFromSystematicIndex(i); + if (propidx != StepProposer::ParameterInPCABlock) { return GetLowerBound(propidx); } else { return -3; } } -double ParameterHandlerBase::GetPCUpperBound(const int i) { - int propidx = parlist.SystematicParameterIndexToPCIndex(i); - if (propidx != ParameterList::ParameterInPCABlock) { +double ParameterHandlerBase::GetProposalUpperBound(const int i) { + int propidx = proposer.GetProposalParameterIndexFromSystematicIndex(i); + if (propidx != StepProposer::ParameterInPCABlock) { return GetUpperBound(propidx); } else { return 3; @@ -572,49 +556,51 @@ double ParameterHandlerBase::GetPCUpperBound(const int i) { void ParameterHandlerBase::SetIndivStepScale(const int ParameterIndex, const double StepScale) { - int idx = parlist.SystematicParameterIndexToPCIndex(ParameterIndex); - if (idx == ParameterList::ParameterInPCABlock) { + int idx = + proposer.GetProposalParameterIndexFromSystematicIndex(ParameterIndex); + if (idx == StepProposer::ParameterInPCABlock) { MACH3LOG_ERROR("You are trying to set the step scale of parameter {}, {}, " "which has been PCA'd.", ParameterIndex, GetParName(ParameterIndex)); throw MaCh3Exception(__FILE__, __LINE__); } - proposer.params.scale[idx] = StepScale; + proposer.proposal_basis.scale[idx] = StepScale; } double ParameterHandlerBase::GetIndivStepScale(const int ParameterIndex) const { - int idx = parlist.SystematicParameterIndexToPCIndex(ParameterIndex); - if (idx == ParameterList::ParameterInPCABlock) { + int idx = + proposer.GetProposalParameterIndexFromSystematicIndex(ParameterIndex); + if (idx == StepProposer::ParameterInPCABlock) { MACH3LOG_ERROR("You are trying to get the step scale of parameter {}, {}, " "which has been PCA'd.", ParameterIndex, GetParName(ParameterIndex)); throw MaCh3Exception(__FILE__, __LINE__); } - return proposer.params.scale[idx]; + return proposer.proposal_basis.scale[idx]; } // ******************************************** void ParameterHandlerBase::SetIndivStepScale( const std::vector &stepscale) { // ******************************************** - if (int(stepscale.size()) != parlist.NumSystematicBasisParameters()) { + if (int(stepscale.size()) != parlist.NumParameters()) { MACH3LOG_WARN( "Stepscale vector not equal to number of parameters. Quitting.."); MACH3LOG_WARN("Size of argument vector: {}", stepscale.size()); - MACH3LOG_WARN("Expected size: {}", parlist.NumSystematicBasisParameters()); + MACH3LOG_WARN("Expected size: {}", parlist.NumParameters()); return; } - for (int i = 0; i < parlist.NumSystematicBasisParameters(); ++i) { + for (int i = 0; i < parlist.NumParameters(); ++i) { - int idx = parlist.SystematicParameterIndexToPCIndex(i); - if (idx == ParameterList::ParameterInPCABlock) { + int idx = proposer.GetProposalParameterIndexFromSystematicIndex(i); + if (idx == StepProposer::ParameterInPCABlock) { continue; } - proposer.params.scale[idx] = stepscale[i]; + proposer.proposal_basis.scale[idx] = stepscale[i]; } PrintIndivStepScale(); @@ -625,14 +611,13 @@ void ParameterHandlerBase::PrintIndivStepScale() const { // ******************************************** MACH3LOG_INFO("============================================================"); MACH3LOG_INFO("{:<{}} | {:<11}", "Parameter:", PrintLength, "Step scale:"); - for (int iParam = 0; iParam < parlist.NumSystematicBasisParameters(); - iParam++) { - int idx = parlist.SystematicParameterIndexToPCIndex(iParam); - if (idx == ParameterList::ParameterInPCABlock) { + for (int iParam = 0; iParam < parlist.NumParameters(); iParam++) { + int idx = proposer.GetProposalParameterIndexFromSystematicIndex(iParam); + if (idx == StepProposer::ParameterInPCABlock) { MACH3LOG_INFO("{:<{}} | {:<11}", GetParName(idx), PrintLength, "PCA'd"); } else { MACH3LOG_INFO("{:<{}} | {:<11}", GetParName(idx), PrintLength, - proposer.params.scale[idx]); + proposer.proposal_basis.scale[idx]); } } MACH3LOG_INFO("============================================================"); @@ -641,8 +626,8 @@ void ParameterHandlerBase::PrintIndivStepScale() const { // ******************************************** void ParameterHandlerBase::ResetIndivStepScale() { // ******************************************** - proposer.params.global_scale = 1; - proposer.params.scale = Eigen::ArrayXd::Ones(proposer.NumParameters()); + proposer.proposal_basis.global_scale = 1; + proposer.proposal_basis.scale.setOnes(); } // ******************************************** @@ -655,22 +640,20 @@ void ParameterHandlerBase::SetThrowMatrix(TMatrixDSym *cov) { Eigen::MatrixXd covariance = M3::ROOTToEigen(*cov); Eigen::MatrixXd covariance_pc = - Eigen::MatrixXd::Zero(proposer.NumParameters(), proposer.NumParameters()); - - covariance_pc.topLeftCorner(parlist.pca.first_index, - parlist.pca.first_index) = - parlist.params.covariance.topLeftCorner(parlist.pca.first_index, - parlist.pca.first_index); - - covariance_pc.block(parlist.pca.first_index, parlist.pca.first_index, - parlist.pca.npc_parameters(), - parlist.pca.npc_parameters()) = - Eigen::MatrixXd::Identity(parlist.pca.npc_parameters(), - parlist.pca.npc_parameters()); - - covariance_pc.bottomRightCorner(parlist.pca.ntail, parlist.pca.ntail) = - parlist.params.covariance.bottomRightCorner(parlist.pca.ntail, - parlist.pca.ntail); + Eigen::MatrixXd::Identity(proposer.NumProposalBasisParameters(), + proposer.NumProposalBasisParameters()); + + covariance_pc.topLeftCorner(proposer.systematic_basis.pca.first_index, + proposer.systematic_basis.pca.first_index) = + parlist.params.covariance.topLeftCorner( + proposer.systematic_basis.pca.first_index, + proposer.systematic_basis.pca.first_index); + + covariance_pc.bottomRightCorner(proposer.systematic_basis.pca.ntail, + proposer.systematic_basis.pca.ntail) = + parlist.params.covariance.bottomRightCorner( + proposer.systematic_basis.pca.ntail, + proposer.systematic_basis.pca.ntail); proposer.SetProposalMatrix(covariance_pc); } diff --git a/Parameters/ParameterHandlerBase.h b/Parameters/ParameterHandlerBase.h index f46201e90..631cf467a 100644 --- a/Parameters/ParameterHandlerBase.h +++ b/Parameters/ParameterHandlerBase.h @@ -202,12 +202,12 @@ class ParameterHandlerBase { std::string GetParName(const int i) const { return parlist.params.name.at(i); } - //need these 'reach into PC' functions for MINUIT - std::string GetPCParName(const int i); - double GetPCParInit(const int i); - double GetPCDiagonalError(const int i); - double GetPCLowerBound(const int i); - double GetPCUpperBound(const int i); + // need these 'reach into PC' functions for MINUIT + std::string GetProposalParName(const int i); + double GetProposalParInit(const int i); + double GetProposalDiagonalError(const int i); + double GetProposalLowerBound(const int i); + double GetProposalUpperBound(const int i); /// @brief Get index based on name /// @ingroup ParameterHandlerGetters @@ -299,10 +299,10 @@ class ParameterHandlerBase { /// @brief Get total number of parameters /// @ingroup ParameterHandlerGetters - int GetNumSystematicParams() const { - return parlist.NumSystematicBasisParameters(); + int GetNumSystematicParams() const { return parlist.NumParameters(); } + int GetNumProposalParams() const { + return proposer.NumProposalBasisParameters(); } - int GetNumProposalParams() const { return parlist.NumPCBasisParameters(); } /// @brief Get the pre-fit values of the parameters. /// @ingroup ParameterHandlerGetters std::vector GetPreFitValues() const { @@ -345,12 +345,14 @@ class ParameterHandlerBase { double GetIndivStepScale(const int ParameterIndex) const; /// @brief Get global step scale for covariance object /// @ingroup ParameterHandlerGetters - double GetGlobalStepScale() const { return proposer.params.global_scale; } + double GetGlobalStepScale() const { + return proposer.proposal_basis.global_scale; + } /// @brief Get number of params which will be different depending if using /// Eigen decomposition or not /// @ingroup ParameterHandlerGetters - int GetNParameters() const { return parlist.NumSystematicBasisParameters(); } + int GetNParameters() const { return parlist.NumParameters(); } /// @brief Print prior value for every parameter void PrintNominal() const; @@ -406,7 +408,7 @@ class ParameterHandlerBase { /// @brief Is parameter fixed or not /// @param i Parameter index bool IsParameterFixed(const int i) const; - bool IsPCParameterFixed(const int i) const; + bool IsProposalParameterFixed(const int i) const; /// @brief Is parameter fixed or not /// @param name Name of parameter you want to check if is fixed @@ -458,5 +460,5 @@ class ParameterHandlerBase { public: ParameterList parlist; - StepProposer proposer; + mutable StepProposer proposer; }; diff --git a/Parameters/ParameterHandlerGeneric.cpp b/Parameters/ParameterHandlerGeneric.cpp index 225fe1fcd..668a6766b 100644 --- a/Parameters/ParameterHandlerGeneric.cpp +++ b/Parameters/ParameterHandlerGeneric.cpp @@ -12,7 +12,7 @@ ParameterHandlerGeneric::ParameterHandlerGeneric( InitParametersTypeFromConfig(); // ETA - again this really doesn't need to be here... - for (int i = 0; i < parlist.NumSystematicBasisParameters(); i++) { + for (int i = 0; i < parlist.NumParameters(); i++) { // Sort out the print length if (int(parlist.params.name[i].length()) > PrintLength) PrintLength = int(parlist.params.name[i].length()); @@ -111,7 +111,7 @@ void ParameterHandlerGeneric::DetermineGlobalParameterIndices() { GlobalParams.push_back(&par); } - //LP order by index... probably doesn't help but probably doesn't hurt + // LP order by index... probably doesn't help but probably doesn't hurt std::sort(GlobalParams.begin(), GlobalParams.end(), [](TypeParameterBase const *l, TypeParameterBase const *r) { return l->index < r->index; @@ -467,7 +467,7 @@ void ParameterHandlerGeneric::IterateOverParams(const std::string &SampleName, // ******************************************** void ParameterHandlerGeneric::InitParams() { // ******************************************** - for (int i = 0; i < parlist.NumSystematicBasisParameters(); ++i) { + for (int i = 0; i < parlist.NumParameters(); ++i) { // ETA - set the name to be xsec_% as this is what ProcessorMCMC expects parlist.params.name[i] = "xsec_" + std::to_string(i); @@ -794,7 +794,7 @@ void ParameterHandlerGeneric::SetGroupOnlyParameters( // ******************************************** // If empty, set the proposed to prior if (Pars.empty()) { - for (int i = 0; i < parlist.NumSystematicBasisParameters(); i++) { + for (int i = 0; i < parlist.NumParameters(); i++) { if (IsParFromGroup(i, Group)) { SetParProp(i, GetParInit(i)); } @@ -807,7 +807,7 @@ void ParameterHandlerGeneric::SetGroupOnlyParameters( throw MaCh3Exception(__FILE__, __LINE__); } int Counter = 0; - for (int i = 0; i < parlist.NumSystematicBasisParameters(); i++) { + for (int i = 0; i < parlist.NumParameters(); i++) { // If belongs to group set value from parsed vector, otherwise use propose // value if (IsParFromGroup(i, Group)) { @@ -823,7 +823,7 @@ void ParameterHandlerGeneric::SetGroupOnlyParameters( void ParameterHandlerGeneric::ToggleFixGroupOnlyParameters( const std::string &Group) { // ******************************************** - for (int i = 0; i < parlist.NumSystematicBasisParameters(); ++i) + for (int i = 0; i < parlist.NumParameters(); ++i) if (IsParFromGroup(i, Group)) ToggleFixParameter(i); } @@ -842,7 +842,7 @@ void ParameterHandlerGeneric::ToggleFixGroupOnlyParameters( void ParameterHandlerGeneric::SetFixGroupOnlyParameters( const std::string &Group) { // ******************************************** - for (int i = 0; i < parlist.NumSystematicBasisParameters(); ++i) + for (int i = 0; i < parlist.NumParameters(); ++i) if (IsParFromGroup(i, Group)) SetFixParameter(i); } @@ -861,7 +861,7 @@ void ParameterHandlerGeneric::SetFixGroupOnlyParameters( void ParameterHandlerGeneric::SetFreeGroupOnlyParameters( const std::string &Group) { // ******************************************** - for (int i = 0; i < parlist.NumSystematicBasisParameters(); ++i) + for (int i = 0; i < parlist.NumParameters(); ++i) if (IsParFromGroup(i, Group)) SetFreeParameter(i); } @@ -898,7 +898,7 @@ int ParameterHandlerGeneric::GetNumParFromGroup( const std::string &Group) const { // ******************************************** int Counter = 0; - for (int i = 0; i < parlist.NumSystematicBasisParameters(); i++) { + for (int i = 0; i < parlist.NumParameters(); i++) { if (IsParFromGroup(i, Group)) Counter++; } @@ -930,24 +930,17 @@ void ParameterHandlerGeneric::DumpMatrixToFile(const std::string &Name) { TObjArray *xsec_spline_interpolation = new TObjArray(); TObjArray *xsec_spline_names = new TObjArray(); - TVectorD *xsec_param_prior = - new TVectorD(parlist.NumSystematicBasisParameters()); - TVectorD *xsec_flat_prior = - new TVectorD(parlist.NumSystematicBasisParameters()); - TVectorD *xsec_stepscale = - new TVectorD(parlist.NumSystematicBasisParameters()); - TVectorD *xsec_param_lb = - new TVectorD(parlist.NumSystematicBasisParameters()); - TVectorD *xsec_param_ub = - new TVectorD(parlist.NumSystematicBasisParameters()); - - TVectorD *xsec_param_knot_weight_lb = - new TVectorD(parlist.NumSystematicBasisParameters()); - TVectorD *xsec_param_knot_weight_ub = - new TVectorD(parlist.NumSystematicBasisParameters()); - TVectorD *xsec_error = new TVectorD(parlist.NumSystematicBasisParameters()); - - for (int i = 0; i < parlist.NumSystematicBasisParameters(); ++i) { + TVectorD *xsec_param_prior = new TVectorD(parlist.NumParameters()); + TVectorD *xsec_flat_prior = new TVectorD(parlist.NumParameters()); + TVectorD *xsec_stepscale = new TVectorD(parlist.NumParameters()); + TVectorD *xsec_param_lb = new TVectorD(parlist.NumParameters()); + TVectorD *xsec_param_ub = new TVectorD(parlist.NumParameters()); + + TVectorD *xsec_param_knot_weight_lb = new TVectorD(parlist.NumParameters()); + TVectorD *xsec_param_knot_weight_ub = new TVectorD(parlist.NumParameters()); + TVectorD *xsec_error = new TVectorD(parlist.NumParameters()); + + for (int i = 0; i < parlist.NumParameters(); ++i) { TObjString *nameObj = new TObjString(GetParFancyName(i).c_str()); xsec_param_names->AddLast(nameObj); diff --git a/Parameters/ParameterList.cpp b/Parameters/ParameterList.cpp index 368d313ba..a1fb98b4c 100644 --- a/Parameters/ParameterList.cpp +++ b/Parameters/ParameterList.cpp @@ -71,19 +71,18 @@ void ParameterList::InsertParameters(int insert_before, params.inv_covariance = Eigen::MatrixXd(); } -void ParameterList::SetParameterCorrelation(int pidi, int pidj, double corr) { - if (pidi == pidj) { +void ParameterList::SetParameterCorrelation(int i, int j, double corr) { + if (i == j) { MACH3LOG_ERROR( "AddParameterCorrelation cannot be used to set covariance " "matrix diagonal elements: ({0},{0}) attempted to be set to {1}", - pidi, corr); + i, corr); throw MaCh3Exception(__FILE__, __LINE__); } - params.covariance(pidi, pidj) = - corr * - std::sqrt(params.covariance(pidi, pidi) * params.covariance(pidj, pidj)); - params.covariance(pidj, pidi) = params.covariance(pidi, pidj); + params.covariance(i, j) = + corr * std::sqrt(params.covariance(i, i) * params.covariance(j, j)); + params.covariance(j, i) = params.covariance(i, j); params.inv_covariance = Eigen::MatrixXd(); } @@ -122,80 +121,6 @@ int ParameterList::FindParameterByFancyName( return int(std::distance(params.fancy_name.begin(), pit)); } -void ParameterList::ConstructTruncatedPCA(double threshold, int first, - int last) { - - M3::EnsureNoOutOfBlockCorrelations(params.covariance, {first, last}); - - pca.first_index = first; - pca.last_index = last; - pca.ntail = int(params.prefit.size() - (last + 1)); - - int blocksize = (last + 1) - first; - - std::tie(pca.pc_to_syst_rotation, pca.syst_to_pc_rotation) = - CalculateTruncatedPCARotation( - params.covariance.block(pca.first_index, pca.first_index, blocksize, - blocksize), - threshold); - - MACH3LOG_INFO("ParameterList::ConstructTruncatedPCA: threshold = {}, " - "nsystparams = {} truncated to {} PC params.", - threshold, pca.nrotated_syst_parameters(), - pca.npc_parameters()); -} - -void ParameterList::RotatePCParameterValuesToSystematicBasis( - Eigen::ArrayXd const &pc_vals, Eigen::ArrayXd &systematic_vals) const { - - systematic_vals.head(pca.first_index) = pc_vals.head(pca.first_index); - - // 1) rotate the principle component parameter values into 'deltas' in the - // systematic basis into the by using the PCA matrix. - // 2) add on the prefit values from to these deltas to return to the - // systematic basis. - auto syst_param_delta = - (pca.pc_to_syst_rotation * - pc_vals.segment(pca.first_index, pca.npc_parameters()).matrix()) - .array(); - - systematic_vals.segment(pca.first_index, pca.nrotated_syst_parameters()) = - params.prefit.segment(pca.first_index, pca.nrotated_syst_parameters()) + - syst_param_delta; - - systematic_vals.tail(pca.ntail) = pc_vals.tail(pca.ntail); -} - -void ParameterList::RotateSystematicParameterValuesToPCBasis( - Eigen::ArrayXd const &systematic_vals, Eigen::ArrayXd &pc_vals) const { - - pc_vals.head(pca.first_index) = systematic_vals.head(pca.first_index); - - // 1) subtract the prefit values from the current parameter values so that the - // PC basis has a CV of 0 - // 2) rotate the resulting parameter 'deltas' in the systematic basis into the - // PC basis by using the transpose of the PCA matrix - auto syst_param_delta = - (systematic_vals - params.prefit) - .segment(pca.first_index, pca.nrotated_syst_parameters()) - .matrix(); - - pc_vals.segment(pca.first_index, pca.npc_parameters()) = - (pca.syst_to_pc_rotation * syst_param_delta).array(); - - pc_vals.tail(pca.ntail) = systematic_vals.tail(pca.ntail); -} - -int ParameterList::SystematicParameterIndexToPCIndex(int i) const { - if (i < pca.first_index) { - return i; - } else if ((i >= pca.first_index) && (i <= pca.last_index)) { - return ParameterInPCABlock; - } else { - return i - pca.nrotated_syst_parameters() + pca.npc_parameters(); - } -} - double ParameterList::Chi2(Eigen::ArrayXd const &systematic_vals) { if (!params.inv_covariance.size()) { @@ -208,46 +133,82 @@ double ParameterList::Chi2(Eigen::ArrayXd const &systematic_vals) { return diff.transpose() * params.inv_covariance * diff; } -StepProposer ParameterList::MakeProposer() const { +StepProposer ParameterList::MakePCAProposer(double threshold, int first, + int last) const { StepProposer proposer; + if ((first >= 0) && (first < params.prefit.size())) { + + if (first >= last) { + MACH3LOG_ERROR("On constructing PCA'd StepProposer, given first index: " + "{} and incompatible last index: {}.", + first, last); + throw MaCh3Exception(__FILE__, __LINE__); + } + + M3::EnsureNoOutOfBlockCorrelations(params.covariance, {first, last}); + + proposer.systematic_basis.pca.first_index = first; + proposer.systematic_basis.pca.last_index = last; + proposer.systematic_basis.pca.ntail = + int(params.prefit.size() - (last + 1)); + proposer.systematic_basis.pca.offset = params.prefit; + + int blocksize = (last + 1) - first; + + std::tie(proposer.systematic_basis.pca.pc_to_syst_rotation, + proposer.systematic_basis.pca.syst_to_pc_rotation) = + CalculateTruncatedPCARotation( + params.covariance.block(proposer.systematic_basis.pca.first_index, + proposer.systematic_basis.pca.first_index, + blocksize, blocksize), + threshold); + + MACH3LOG_INFO( + "ParameterList::MakePCAProposer: threshold = {}, " + "nsystparams = {} truncated to {} PC params.", + threshold, + proposer.systematic_basis.pca.nrotated_systematic_parameters(), + proposer.systematic_basis.pca.npc_parameters()); + } else { + proposer.systematic_basis.pca.first_index = NumParameters(); + proposer.systematic_basis.pca.last_index = NumParameters(); + proposer.systematic_basis.pca.ntail = 0; + } // if we are using a PCA rotation then the proposer should only be aware of // the PC parameter basis. // copy the unrotated parameters and covariance blocks - int nproposer_parameters = NumPCBasisParameters(); - - Eigen::ArrayXd prefit_pc = Eigen::ArrayXd::Zero(nproposer_parameters); - - RotateSystematicParameterValuesToPCBasis(params.prefit, prefit_pc); + int nproposer_parameters = proposer.NumProposalBasisParameters(); - proposer.SetParameterValues(prefit_pc); + proposer.SetSystematicParameterValues(params.prefit); Eigen::MatrixXd covariance_pc = - Eigen::MatrixXd::Zero(nproposer_parameters, nproposer_parameters); + Eigen::MatrixXd::Identity(nproposer_parameters, nproposer_parameters); - covariance_pc.topLeftCorner(pca.first_index, pca.first_index) = - params.covariance.topLeftCorner(pca.first_index, pca.first_index); + covariance_pc.topLeftCorner(proposer.systematic_basis.pca.first_index, + proposer.systematic_basis.pca.first_index) = + params.covariance.topLeftCorner( + proposer.systematic_basis.pca.first_index, + proposer.systematic_basis.pca.first_index); - covariance_pc.block(pca.first_index, pca.first_index, pca.npc_parameters(), - pca.npc_parameters()) = - Eigen::MatrixXd::Identity(pca.npc_parameters(), pca.npc_parameters()); - - covariance_pc.bottomRightCorner(pca.ntail, pca.ntail) = - params.covariance.bottomRightCorner(pca.ntail, pca.ntail); + covariance_pc.bottomRightCorner(proposer.systematic_basis.pca.ntail, + proposer.systematic_basis.pca.ntail) = + params.covariance.bottomRightCorner(proposer.systematic_basis.pca.ntail, + proposer.systematic_basis.pca.ntail); proposer.SetProposalMatrix(covariance_pc); // PCA parameters get a step scale of 1 - proposer.params.scale = Eigen::ArrayXd::Ones(NumPCBasisParameters()); - proposer.params.isfree = Eigen::ArrayXi::Ones(NumPCBasisParameters()); + proposer.proposal_basis.scale = Eigen::ArrayXd::Ones(nproposer_parameters); + proposer.proposal_basis.isfree = Eigen::ArrayXi::Ones(nproposer_parameters); - for (int i = 0; i < NumSystematicBasisParameters(); ++i) { + for (int i = 0; i < NumParameters(); ++i) { - int idx = SystematicParameterIndexToPCIndex(i); + int idx = proposer.GetProposalParameterIndexFromSystematicIndex(i); if (params.flip_pivot[i].first) { - if (idx == ParameterInPCABlock) { + if (idx == StepProposer::ParameterInPCABlock) { MACH3LOG_ERROR("Parameter, {}, index: {} is in the PCA block but " "contained a special flip proposal definition", params.name[i], i); @@ -259,7 +220,7 @@ StepProposer ParameterList::MakeProposer() const { } if (std::get<0>(params.circ_bounds[i])) { - if (idx == ParameterInPCABlock) { + if (idx == StepProposer::ParameterInPCABlock) { MACH3LOG_ERROR( "Parameter, {}, index: {} is in the PCA block but " "contained a special circular bounds proposal definition", @@ -271,30 +232,33 @@ StepProposer ParameterList::MakeProposer() const { std::get<2>(params.circ_bounds[i])}); } - if (idx == ParameterInPCABlock) { + if (idx == StepProposer::ParameterInPCABlock) { continue; } - proposer.params.scale[idx] = params.stepscale[i]; - proposer.params.isfree[idx] = params.isfree[i]; + proposer.proposal_basis.scale[idx] = params.stepscale[i]; + proposer.proposal_basis.isfree[idx] = params.isfree[i]; } - proposer.params.global_scale = 1; + proposer.proposal_basis.global_scale = 1; return proposer; } +StepProposer ParameterList::MakeProposer() const { + return MakePCAProposer(0, std::numeric_limits::max(), 0); +} + std::string ParameterList::SystematicParameterToString(int i) const { return fmt::format( R"(name: {}, fancy_name: {} - prefit: {:.3g}, error: {:.3g}, bounds: [ {:.3g}, {:.3g} ], stepscale: {:.3g} - flatprior: {}, isfree: {} - is_pca: {}, has_flip: {}, has_circ_bounds: {})", + prefit: {:.3g}, error: {:.3g}, bounds: [ {:.3g}, {:.3g} ], + stepscale: {:.3g} + flatprior: {}, isfree: {} has_flip: {}, has_circ_bounds: {})", params.name[i], params.fancy_name[i], params.prefit[i], params.error[i], params.lowbound[i], params.upbound[i], params.stepscale[i], - params.flatprior[i], params.isfree[i], - SystematicParameterIndexToPCIndex(i) == ParameterInPCABlock, - params.flip_pivot[i].first, std::get<0>(params.circ_bounds[i])); + params.flatprior[i], params.isfree[i], params.flip_pivot[i].first, + std::get<0>(params.circ_bounds[i])); } ParameterList ParameterList::MakeFromYAML(YAML::Node const &config) { @@ -476,10 +440,6 @@ ParameterList ParameterList::MakeFromYAML(YAML::Node const &config) { parlist.params.covariance = M3::MakeMatrixPosDef(parlist.params.covariance); - parlist.pca.first_index = parlist.NumSystematicBasisParameters(); - parlist.pca.last_index = parlist.NumSystematicBasisParameters(); - parlist.pca.ntail = 0; - MACH3LOG_INFO("----------------"); MACH3LOG_INFO("Found {} systematics parameters in total", parlist.params.prefit.size()); @@ -558,9 +518,5 @@ ParameterList ParameterList::MakeFromTMatrix(std::string const &name, parlist.params.covariance = M3::MakeMatrixPosDef(M3::ROOTToEigen(*CovMat)); - parlist.pca.first_index = parlist.NumSystematicBasisParameters(); - parlist.pca.last_index = parlist.NumSystematicBasisParameters(); - parlist.pca.ntail = 0; - return parlist; } diff --git a/Parameters/ParameterList.h b/Parameters/ParameterList.h index 852c9d452..d6f264f88 100644 --- a/Parameters/ParameterList.h +++ b/Parameters/ParameterList.h @@ -60,51 +60,26 @@ struct ParameterList { } void AddParameters(std::vector const &new_params) { - InsertParameters(NumSystematicBasisParameters(), new_params); + InsertParameters(NumParameters(), new_params); } void AddParameter(ParamInfo const &new_param) { - InsertParameter(NumSystematicBasisParameters(), new_param); + InsertParameter(NumParameters(), new_param); } - std::string SystematicParameterToString(int i) const; - - void SetParameterCorrelation(int pidi, int pidj, double corr); + void SetParameterCorrelation(int i, int j, double corr); void SetParameterAllCorrelations( int paramid, std::map const &correlations); int FindParameter(std::string const &name) const; int FindParameterByFancyName(std::string const &fancy_name) const; - int NumSystematicBasisParameters() const { return int(params.prefit.size()); } - int NumPCBasisParameters() const { - return NumSystematicBasisParameters() - pca.nrotated_syst_parameters() + - pca.npc_parameters(); - } - - struct { - int first_index, last_index, ntail; - Eigen::MatrixXd pc_to_syst_rotation; - Eigen::MatrixXd syst_to_pc_rotation; - int nrotated_syst_parameters() const { - return int(pc_to_syst_rotation.rows()); - } - int npc_parameters() const { return int(pc_to_syst_rotation.cols()); } - } pca; - - void ConstructTruncatedPCA(double threshold, int first, int last); - - void RotatePCParameterValuesToSystematicBasis( - Eigen::ArrayXd const &pc_vals, Eigen::ArrayXd &systematic_vals) const; - - void RotateSystematicParameterValuesToPCBasis( - Eigen::ArrayXd const &systematic_vals, Eigen::ArrayXd &pc_vals) const; - - constexpr static int ParameterInPCABlock = std::numeric_limits::max(); - int SystematicParameterIndexToPCIndex(int i) const; + int NumParameters() const { return int(params.prefit.size()); } + std::string SystematicParameterToString(int i) const; double Chi2(Eigen::ArrayXd const &systematic_vals); StepProposer MakeProposer() const; + StepProposer MakePCAProposer(double threshold, int first, int last) const; struct { /// The input root file we read in diff --git a/Parameters/README.md b/Parameters/README.md index a30166e14..66b36d913 100644 --- a/Parameters/README.md +++ b/Parameters/README.md @@ -5,7 +5,6 @@ Elementary objects needed for sampling: - ParameterList - Read from YAML. Knows everything about each systematic parameter: name, error, possibly-correlated prior, restrictions on parameter applicability etc... - - Optionally performs basis translation and dimensionality truncation via block diagonal PCA and retains the transformations between the 'Systematic Parameter Basis' and the 'Proposer Parameter Basis' (possibly PCA'd). - StepProposer - Knows nothing about the meaning of parameters. @@ -13,7 +12,7 @@ Elementary objects needed for sampling: - Keeps track of parameter step scales and possibly updates the proposal matrix as steps are accepted. - Possibly steps in a rotated parameter space. - Possibly has 'special proposals': flips and circular - - Does not need to know how to translate from its parameter basis back to the systematic parameter basis, that is the job of ParameterList + - Needs to know how to translate from its parameter basis back to the systematic parameter basis, that is the job of ParameterList Open Questions: diff --git a/Parameters/StepProposer.cpp b/Parameters/StepProposer.cpp index 0d3449ccb..5c840810c 100644 --- a/Parameters/StepProposer.cpp +++ b/Parameters/StepProposer.cpp @@ -6,60 +6,151 @@ StepProposer::StepProposer() { rng.gaus = std::normal_distribution(0); rng.unif = std::uniform_real_distribution(0, 1); + + proposal_basis._propid = 0; + systematic_basis._propid = std::numeric_limits::max(); }; StepProposer::StepProposer(Eigen::MatrixXd proposal_matrix, Eigen::ArrayXd current_values) : StepProposer() { SetProposalMatrix(proposal_matrix); - SetParameterValues(current_values); + SetSystematicParameterValues(current_values); +} + +int StepProposer::GetProposalParameterIndexFromSystematicIndex(int i) const { + if (i < systematic_basis.pca.first_index) { + return i; + } else if ((i >= systematic_basis.pca.first_index) && + (i <= systematic_basis.pca.last_index)) { + return ParameterInPCABlock; + } else { + return i - systematic_basis.pca.nrotated_systematic_parameters() + + systematic_basis.pca.npc_parameters(); + } +} + +void StepProposer::SetProposalMatrix(Eigen::MatrixXd proposal_matrix) { + proposal_basis.proposal = proposal_matrix; + Eigen::LLT lltproposal(proposal_basis.proposal); + if (lltproposal.info() != Eigen::Success) { + MACH3LOG_ERROR("Failed to LLT decompose proposal matrix"); + throw MaCh3Exception(__FILE__, __LINE__); + } + proposal_basis.l_proposal = lltproposal.matrixL(); +} + +void StepProposer::SetSystematicParameterValues( + Eigen::ArrayXd const &new_values) { + systematic_basis.proposed = new_values; + TranformSystematicParametersProposalToBasis(systematic_basis.proposed, + proposal_basis.proposed); + systematic_basis._propid = 0; + proposal_basis._propid = 0; +} +void StepProposer::GetSystematicParameterValues( + Eigen::ArrayXd &proposed_values) { + if (systematic_basis._propid != proposal_basis._propid) { + TranformProposalParametersToSystematicBasis(proposal_basis.proposed, + systematic_basis.proposed); + systematic_basis._propid = proposal_basis._propid; + } + proposed_values = systematic_basis.proposed; +} + +void StepProposer::TranformProposalParametersToSystematicBasis( + Eigen::ArrayXd const &proposal_basis_vals, + Eigen::ArrayXd &systematic_basis_vals) const { + systematic_basis_vals.head(systematic_basis.pca.first_index) = + proposal_basis_vals.head(systematic_basis.pca.first_index); + + // 1) rotate the principle component parameter values into 'deltas' in the + // systematic basis into the by using the PCA matrix. + // 2) add on the offset (prefit) values from to these deltas to return to the + // systematic basis. + auto syst_param_delta = (systematic_basis.pca.pc_to_syst_rotation * + proposal_basis_vals + .segment(systematic_basis.pca.first_index, + systematic_basis.pca.npc_parameters()) + .matrix()) + .array(); + + systematic_basis_vals.segment( + systematic_basis.pca.first_index, + systematic_basis.pca.nrotated_systematic_parameters()) = + systematic_basis.pca.offset.segment( + systematic_basis.pca.first_index, + systematic_basis.pca.nrotated_systematic_parameters()) + + syst_param_delta; + + systematic_basis_vals.tail(systematic_basis.pca.ntail) = + proposal_basis_vals.tail(systematic_basis.pca.ntail); +} + +void StepProposer::TranformSystematicParametersProposalToBasis( + Eigen::ArrayXd const &systematic_basis_vals, + Eigen::ArrayXd &proposal_basis_vals) const { + proposal_basis_vals.head(systematic_basis.pca.first_index) = + systematic_basis_vals.head(systematic_basis.pca.first_index); + + // 1) subtract the offset (prefit) values from the current parameter values so + // that the PC basis has a CV of 0 + // 2) rotate the resulting parameter 'deltas' in the systematic basis into the + // PC basis by using the transpose of the PCA matrix + auto syst_param_delta = + (systematic_basis_vals - systematic_basis.pca.offset) + .segment(systematic_basis.pca.first_index, + systematic_basis.pca.nrotated_systematic_parameters()) + .matrix(); + + proposal_basis_vals.segment(systematic_basis.pca.first_index, + systematic_basis.pca.npc_parameters()) = + (systematic_basis.pca.syst_to_pc_rotation * syst_param_delta).array(); + + proposal_basis_vals.tail(systematic_basis.pca.ntail) = + systematic_basis_vals.tail(systematic_basis.pca.ntail); } Eigen::ArrayXd const &StepProposer::Propose() { Eigen::VectorXd random_vector = - Eigen::VectorXd::NullaryExpr(params.current.size(), [&](int i) { - return params.isfree[i] ? rng.gaus(rng.e1) : 0; + Eigen::VectorXd::NullaryExpr(proposal_basis.current.size(), [&](int i) { + return proposal_basis.isfree[i] ? rng.gaus(rng.e1) : 0; }); - params.proposed = - params.current + ((params.l_proposal * random_vector).array() * - params.scale * params.global_scale); + proposal_basis.proposed = + proposal_basis.current + + ((proposal_basis.l_proposal * random_vector).array() * + proposal_basis.scale * proposal_basis.global_scale); + + proposal_basis._propid++; // if we don't have any special proposal functions, leave early if (!special_proposal.enabled) { - return params.proposed; + return proposal_basis.proposed; } for (auto const &[i, low, up] : special_proposal.circ_bounds) { - if (params.isfree[i]) { + if (proposal_basis.isfree[i]) { continue; } - if (params.proposed[i] > up) { - params.proposed[i] = low + std::fmod(params.proposed[i] - up, up - low); - } else if (params.proposed[i] < low) { - params.proposed[i] = up - std::fmod(low - params.proposed[i], up - low); + if (proposal_basis.proposed[i] > up) { + proposal_basis.proposed[i] = + low + std::fmod(proposal_basis.proposed[i] - up, up - low); + } else if (proposal_basis.proposed[i] < low) { + proposal_basis.proposed[i] = + up - std::fmod(low - proposal_basis.proposed[i], up - low); } } for (auto const &[i, pivot] : special_proposal.flips) { - if (params.isfree[i]) { + if (proposal_basis.isfree[i]) { continue; } if (rng.unif(rng.e1) < 0.5) { - params.proposed[i] = 2 * pivot - params.proposed[i]; + proposal_basis.proposed[i] = 2 * pivot - proposal_basis.proposed[i]; } } - return params.proposed; -} - -void StepProposer::SetProposalMatrix(Eigen::MatrixXd proposal_matrix) { - params.proposal = proposal_matrix; - Eigen::LLT lltproposal(params.proposal); - if (lltproposal.info() != Eigen::Success) { - MACH3LOG_ERROR("Failed to LLT decompose proposal matrix"); - throw MaCh3Exception(__FILE__, __LINE__); - } - params.l_proposal = lltproposal.matrixL(); + return proposal_basis.proposed; } diff --git a/Parameters/StepProposer.h b/Parameters/StepProposer.h index b3c75cccf..b65f44d31 100644 --- a/Parameters/StepProposer.h +++ b/Parameters/StepProposer.h @@ -20,10 +20,37 @@ struct StepProposer { struct { Eigen::ArrayXd current, proposed, scale; Eigen::ArrayXi isfree; - double global_scale; Eigen::MatrixXd proposal; Eigen::MatrixXd l_proposal; - } params; + double global_scale; + + // Used to keep track of if we have already transformed the current proposed + // parameters to the systematic_basis + size_t _propid; + + } proposal_basis; + + struct { + Eigen::ArrayXd proposed; + + // Used to keep track of if we have already transformed the current proposed + // parameters to the systematic_basis + size_t _propid; + + Eigen::ArrayXi isfree; + + struct { + int first_index, last_index, ntail; + Eigen::ArrayXd offset; + Eigen::MatrixXd pc_to_syst_rotation; + Eigen::MatrixXd syst_to_pc_rotation; + int nrotated_systematic_parameters() const { + return int(pc_to_syst_rotation.rows()); + } + int npc_parameters() const { return int(pc_to_syst_rotation.cols()); } + } pca; + + } systematic_basis; struct { bool enabled; @@ -50,14 +77,29 @@ struct StepProposer { StepProposer(); StepProposer(Eigen::MatrixXd proposal_matrix, Eigen::ArrayXd current_values); - int NumParameters() const { return int(params.current.size()); } + int NumSystematicBasisParameters() const { + return int(proposal_basis.current.size()); + } + int NumProposalBasisParameters() const { + return NumSystematicBasisParameters() - + systematic_basis.pca.nrotated_systematic_parameters() + + systematic_basis.pca.npc_parameters(); + } void SetProposalMatrix(Eigen::MatrixXd proposal_matrix); - void SetParameterValues(Eigen::ArrayXd current_values) { - params.current = current_values; - } - void EnableAdaption(YAML::Node const &){} + void SetSystematicParameterValues(Eigen::ArrayXd const &new_values); + void GetSystematicParameterValues(Eigen::ArrayXd &proposed_values); + + void TranformProposalParametersToSystematicBasis( + Eigen::ArrayXd const &pc_vals, Eigen::ArrayXd &systematic_vals) const; + void TranformSystematicParametersProposalToBasis( + Eigen::ArrayXd const &systematic_vals, Eigen::ArrayXd &pc_vals) const; + + constexpr static int ParameterInPCABlock = std::numeric_limits::max(); + int GetProposalParameterIndexFromSystematicIndex(int i) const; + + void EnableAdaption(YAML::Node const &) {} // void SetAdaptionCovariance(Eigen::MatrixXd parameter_covariance, // size_t nsteps) { // adaptive.nsteps = nsteps; @@ -65,5 +107,5 @@ struct StepProposer { // } Eigen::ArrayXd const &Propose(); - void Accept() { params.current = params.proposed; } + void Accept() { proposal_basis.current = proposal_basis.proposed; } }; From 1c8fd0fbf46b933161c0ef382ea611d3d34e8356 Mon Sep 17 00:00:00 2001 From: Luke Pickering Date: Fri, 19 Dec 2025 11:01:09 +0000 Subject: [PATCH 13/14] MaCh3_DUNE gets sad about some eigen usage here... --- Parameters/PCA.h | 1 + Parameters/README.md | 1 - Parameters/StepProposer.h | 3 +++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Parameters/PCA.h b/Parameters/PCA.h index 3dc66feb3..d9dae21fd 100644 --- a/Parameters/PCA.h +++ b/Parameters/PCA.h @@ -7,6 +7,7 @@ #pragma GCC diagnostic ignored "-Wconversion" #pragma GCC diagnostic ignored "-Wduplicated-branches" #pragma GCC diagnostic ignored "-Wstrict-aliasing" +#pragma GCC diagnostic ignored "-Wnull-dereference" #include "Eigen/Dense" #pragma GCC diagnostic pop diff --git a/Parameters/README.md b/Parameters/README.md index 66b36d913..5e0250cc2 100644 --- a/Parameters/README.md +++ b/Parameters/README.md @@ -14,6 +14,5 @@ Elementary objects needed for sampling: - Possibly has 'special proposals': flips and circular - Needs to know how to translate from its parameter basis back to the systematic parameter basis, that is the job of ParameterList - Open Questions: - Should the likelihood be calculated in the systematic (full) parameter space or the proposal (possibly truncated) parameter space. NPar might differ significantly? diff --git a/Parameters/StepProposer.h b/Parameters/StepProposer.h index b65f44d31..b1ace7ac7 100644 --- a/Parameters/StepProposer.h +++ b/Parameters/StepProposer.h @@ -9,6 +9,9 @@ #pragma GCC diagnostic ignored "-Wold-style-cast" #pragma GCC diagnostic ignored "-Wuseless-cast" #pragma GCC diagnostic ignored "-Wconversion" +#pragma GCC diagnostic ignored "-Wduplicated-branches" +#pragma GCC diagnostic ignored "-Wstrict-aliasing" +#pragma GCC diagnostic ignored "-Wnull-dereference" #include "Eigen/Dense" #pragma GCC diagnostic pop From 0b091cdee47e8829bb0588b12dba321ca7e9994f Mon Sep 17 00:00:00 2001 From: Luke Pickering Date: Fri, 19 Dec 2025 14:23:22 +0000 Subject: [PATCH 14/14] Expose InsertParameters at the Generic level and fix global indices --- Parameters/ParameterHandlerGeneric.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Parameters/ParameterHandlerGeneric.h b/Parameters/ParameterHandlerGeneric.h index 728e0e566..96e1fd045 100644 --- a/Parameters/ParameterHandlerGeneric.h +++ b/Parameters/ParameterHandlerGeneric.h @@ -196,6 +196,13 @@ class ParameterHandlerGeneric : public ParameterHandlerBase { std::vector GetOscParsFromSampleName(const std::string &SampleName); + void + InsertParameters(int insert_at, + std::vector const &new_params) { + parlist.InsertParameters(insert_at, new_params); + DetermineGlobalParameterIndices(); + } + void DetermineGlobalParameterIndices(); protected: