diff --git a/src/mains/RunCRTM.h b/src/mains/RunCRTM.h index bce6d4e45..9b2e38f61 100644 --- a/src/mains/RunCRTM.h +++ b/src/mains/RunCRTM.h @@ -76,6 +76,11 @@ template class RunCRTM : public oops::Application { ObsDiags_ diag(obsdb[jj], hop.locations(), diagvars); hop.simulateObs(gval, hofx, ybias, qcflags, bias, diag); + + const double zz = hofx.rms(); + const double xx = conf[jj].getDouble("rms ref"); + const double tol = conf[jj].getDouble("tolerance"); +// BOOST_CHECK_CLOSE(xx, zz, tol); } return 0; diff --git a/src/mains/RunPreProcess.h b/src/mains/RunPreProcess.h index 955e6e4f1..ad085ddcd 100644 --- a/src/mains/RunPreProcess.h +++ b/src/mains/RunPreProcess.h @@ -57,8 +57,10 @@ class RunPreProcess : public oops::Application { util::TimeWindow(params.timeWindow.value()), this->getComm()); - ioda::ObsDataVector obserr_(obsdb, obsdb.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags_(obsdb, obsdb.obsvariables()); + std::shared_ptr> obserr_ = + std::make_shared>(obsdb, obsdb.obsvariables(), "ObsError"); + std::shared_ptr> qcflags_ = + std::make_shared>(obsdb, obsdb.obsvariables()); ufo::ObsFilters filters(obsdb, params.filtersParams.toConfiguration(), diff --git a/src/ufo/CMakeLists.txt b/src/ufo/CMakeLists.txt index 6d5e90d5a..2a9ab69f5 100644 --- a/src/ufo/CMakeLists.txt +++ b/src/ufo/CMakeLists.txt @@ -195,7 +195,7 @@ if(geos-aero_FOUND) endif() if(gsw_FOUND) - target_link_libraries(ufo PUBLIC gsw ${gsw_LIBRARIES}) + target_link_libraries(ufo PUBLIC gsw) target_compile_definitions(ufo PUBLIC GSW_FOUND) endif() diff --git a/src/ufo/ObsFilter.cc b/src/ufo/ObsFilter.cc index aa0ef22c1..e963c17ef 100644 --- a/src/ufo/ObsFilter.cc +++ b/src/ufo/ObsFilter.cc @@ -21,7 +21,7 @@ namespace ufo { ObsFilter::ObsFilter(ioda::ObsSpace & os, const ObsFilterParametersBase & parameters, - ioda::ObsDataVector & flags, ioda::ObsDataVector & obserr) + ObsDataPtr_ flags, ObsDataPtr_ obserr) : ofilt_(), filterName_("ObsFilter::"+parameters.filter.value().value()) { oops::Log::trace() << "ObsFilter::ObsFilter starting" << std::endl; diff --git a/src/ufo/ObsFilter.h b/src/ufo/ObsFilter.h index 844c6f9f5..898443855 100644 --- a/src/ufo/ObsFilter.h +++ b/src/ufo/ObsFilter.h @@ -30,6 +30,8 @@ class ObsDiagnostics; /// \brief A filter processing observations. class ObsFilter : public util::Printable, private util::ObjectCounter { + template using ObsDataPtr_ = std::shared_ptr >; + public: static const std::string classname() {return "ufo::ObsFilter";} @@ -39,13 +41,13 @@ class ObsFilter : public util::Printable, /// Space containing the observations to process. /// \param params /// The filter's configuration parameters. - /// \param flags + /// \param qcflags /// Quality control flags. They may be modified by the filter. - /// \param obserr + /// \param obserrors /// Estimates of the standard deviations of observation errors. They may be modified by the /// filter. ObsFilter(ioda::ObsSpace &obsspace, const ObsFilterParametersBase ¶ms, - ioda::ObsDataVector & flags, ioda::ObsDataVector & obserr); + ObsDataPtr_ qcflags, ObsDataPtr_ obserrors); ObsFilter(const ObsFilter &) = delete; ObsFilter(ObsFilter &&) = default; ObsFilter& operator=(const ObsFilter &) = delete; diff --git a/src/ufo/ObsFilterBase.cc b/src/ufo/ObsFilterBase.cc index 1f797072c..767b093f6 100644 --- a/src/ufo/ObsFilterBase.cc +++ b/src/ufo/ObsFilterBase.cc @@ -33,7 +33,7 @@ FilterFactory::FilterFactory(const std::string & name) { std::unique_ptr FilterFactory::create(ioda::ObsSpace & os, const ObsFilterParametersBase & params, - ioda::ObsDataVector & flags, ioda::ObsDataVector & obserr) { + ObsDataPtr_ flags, ObsDataPtr_ obserr) { oops::Log::trace() << "FilterFactory::create starting" << std::endl; const std::string &id = params.filter.value().value(); typename std::map::iterator diff --git a/src/ufo/ObsFilterBase.h b/src/ufo/ObsFilterBase.h index bdc716fc3..b18609826 100644 --- a/src/ufo/ObsFilterBase.h +++ b/src/ufo/ObsFilterBase.h @@ -59,7 +59,7 @@ enum class FilterStage {AUTO, /// In the former case, they should provide a constructor with the following signature: /// /// ObsFilter(ObsSpace_ &, const eckit::Configuration &, -/// ioda::ObsDataVector &, ioda::ObsDataVector &); +/// ObsDataPtr_, ObsDataPtr_); /// /// In the latter case, the implementer should first define a subclass of ObsFilterParametersBase /// holding the settings of the filter in question. The implementation of the ObsFilter interface @@ -67,7 +67,7 @@ enum class FilterStage {AUTO, /// the following signature: /// /// ObsFilter(ObsSpace_ &, const Parameters_ &, -/// ioda::ObsDataVector &, ioda::ObsDataVector &); +/// ObsDataPtr_, ObsDataPtr_); // TODO(someone): check whether we can just keep Parameters here and remove the // Configuration option. class ObsFilterBase : public util::Printable, @@ -136,6 +136,8 @@ class ObsFilterParametersWrapper : public oops::Parameters { /// ObsFilter factory class FilterFactory { + template using ObsDataPtr_ = std::shared_ptr >; + public: /// \brief Create and return a new observation filter. /// @@ -144,8 +146,8 @@ class FilterFactory { /// otherwise an exception will be thrown. static std::unique_ptr create(ioda::ObsSpace &, const ObsFilterParametersBase & params, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + ObsDataPtr_ flags = ObsDataPtr_(), + ObsDataPtr_ obserr = ObsDataPtr_()); /// \brief Create and return an instance of the subclass of ObsFilterParametersBase /// storing parameters of observation filters of the specified type. @@ -167,8 +169,7 @@ class FilterFactory { private: virtual std::unique_ptr make(ioda::ObsSpace &, const ObsFilterParametersBase &, - ioda::ObsDataVector &, - ioda::ObsDataVector &) = 0; + ObsDataPtr_, ObsDataPtr_) = 0; virtual std::unique_ptr makeParameters() const = 0; @@ -187,10 +188,12 @@ class FilterMaker : public FilterFactory { typedef oops::TParameters_IfAvailableElseFallbackType_t Parameters_; + template using ObsDataPtr_ = std::shared_ptr >; + std::unique_ptr make(ioda::ObsSpace & os, const ObsFilterParametersBase & params, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) override { + ObsDataPtr_ flags, + ObsDataPtr_ obserr) override { const auto &stronglyTypedParams = dynamic_cast(params); return std::make_unique(os, oops::parametersOrConfiguration::value>(stronglyTypedParams), diff --git a/src/ufo/ObsFilters.cc b/src/ufo/ObsFilters.cc index 1aaa7b6d9..e5d23fd72 100644 --- a/src/ufo/ObsFilters.cc +++ b/src/ufo/ObsFilters.cc @@ -22,11 +22,10 @@ namespace ufo { ObsFilters::ObsFilters(ioda::ObsSpace & os, const eckit::Configuration & config, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr, + ObsDataPtr_ qcflags, ObsDataPtr_ obserr, const int iteration) - : obsspace_(os), qcmanager_(nullptr), autoFilters_(), preFilters_(), priorFilters_(), - postFilters_(), geovars_(), diagvars_(), qcflags_(flags), obserr_(obserr), + : obsspace_(os), autoFilters_(), preFilters_(), priorFilters_(), postFilters_(), + geovars_(), diagvars_(), qcflags_(qcflags), obserrfilter_(obserr), iteration_(iteration) { oops::Log::trace() << "ObsFilters::ObsFilters starting" << std::endl; @@ -52,21 +51,22 @@ ObsFilters::ObsFilters(ioda::ObsSpace & os, // If at least one filter has been configured, prepend the QC manager and // append the Final Check to the list of filters. - atLeastOneFilterConfigured_ = - atLeastOneAutoFilterConfigured || atLeastOnePrePriorPostFilterConfigured; + const bool atLeastOneFilterConfigured = + atLeastOneAutoFilterConfigured || atLeastOnePrePriorPostFilterConfigured; - // Prepare QC handling and statistics if any filters are present - if (atLeastOneFilterConfigured_) { +// Prepare QC handling and statistics if any filters are present + if (atLeastOneFilterConfigured) { eckit::LocalConfiguration conf; conf.set("filter", "QCmanager"); ObsFilterParametersWrapper filterParams; filterParams.validateAndDeserialize(conf); - qcmanager_ = std::make_unique(os, filterParams.filterParameters, - qcflags_, obserr_); + if (atLeastOneAutoFilterConfigured) + autoFilters_.emplace_back(os, filterParams.filterParameters, qcflags_, obserrfilter_); + else + preFilters_.emplace_back(os, filterParams.filterParameters, qcflags_, obserrfilter_); } - // Create the filters, only at 0-th iteration, or at iterations specified in - // "apply at iterations" +// Create the filters, only at 0-th iteration, or at iterations specified in "apply at iterations" if (atLeastOneAutoFilterConfigured) appendToFiltersList(autoFiltersParams, autoFilters_); if (atLeastOnePrePriorPostFilterConfigured) { @@ -75,18 +75,16 @@ ObsFilters::ObsFilters(ioda::ObsSpace & os, appendToFiltersList(postFiltersParams, postFilters_); } - // Create the final filter run at the end of the pipeline - if (atLeastOneFilterConfigured_) { +// Create the final filter run at the end of the pipeline + if (atLeastOneFilterConfigured) { eckit::LocalConfiguration conf; conf.set("filter", "Final Check"); ObsFilterParametersWrapper filterParams; filterParams.validateAndDeserialize(conf); if (atLeastOneAutoFilterConfigured) - autoFilters_.emplace_back(os, filterParams.filterParameters, qcflags_, - obserr_); + autoFilters_.emplace_back(os, filterParams.filterParameters, qcflags_, obserrfilter_); else - postFilters_.emplace_back(os, filterParams.filterParameters, qcflags_, - obserr_); + postFilters_.emplace_back(os, filterParams.filterParameters, qcflags_, obserrfilter_); } oops::Log::trace() << "ObsFilters::ObsFilters done" << std::endl; @@ -94,20 +92,18 @@ ObsFilters::ObsFilters(ioda::ObsSpace & os, // ----------------------------------------------------------------------------- -void ObsFilters::appendToFiltersList(const FilterParams_& filtersParams, - std::vector& filters) { - for (const ObsFilterParametersWrapper& filterParams : filtersParams) { +void ObsFilters::appendToFiltersList(const FilterParams_ & filtersParams, + std::vector & filters) { + for (const ObsFilterParametersWrapper & filterParams : filtersParams) { // Only create filters for the 0-th iteration by default bool apply = (iteration_ == 0); // If "apply at iterations" is set, check if this is the right iteration if (filterParams.applyAtIterations.value() != boost::none) { - const std::set iters = - oops::parseIntSet(*filterParams.applyAtIterations.value()); + const std::set iters = oops::parseIntSet(*filterParams.applyAtIterations.value()); apply = oops::contains(iters, iteration_); } if (apply) { - filters.emplace_back(obsspace_, filterParams.filterParameters, - qcflags_, obserr_); + filters.emplace_back(obsspace_, filterParams.filterParameters, qcflags_, obserrfilter_); geovars_ += filters.back().requiredVars(); diagvars_ += filters.back().requiredHdiagnostics(); } @@ -117,78 +113,71 @@ void ObsFilters::appendToFiltersList(const FilterParams_& filtersParams, // ----------------------------------------------------------------------------- void ObsFilters::preProcess() { - if (atLeastOneFilterConfigured_) { - if (autoFilters_.size() > 0) { - qcmanager_->preProcess(); // No operation, placeholder - for (ObsFilter& filter : autoFilters_) { - filter.checkFilterData(FilterStage::AUTO); - filter.preProcess(); - } - } else { - qcmanager_->preProcess(); // No operation, placeholder - for (ObsFilter& filter : preFilters_) { - filter.checkFilterData(FilterStage::PRE); - filter.preProcess(); - } + if (autoFilters_.size() > 0) { + for (ObsFilter & filter : autoFilters_) { + filter.checkFilterData(FilterStage::AUTO); + filter.preProcess(); + } + } else { + for (ObsFilter & filter : preFilters_) { + filter.checkFilterData(FilterStage::PRE); + filter.preProcess(); } } } // ----------------------------------------------------------------------------- -void ObsFilters::priorFilter(const GeoVaLs& gv) { - if (atLeastOneFilterConfigured_) { - qcmanager_->priorFilter(gv); // No operation, placeholder - if (autoFilters_.size() > 0) { - for (ObsFilter& filter : autoFilters_) { - filter.checkFilterData(FilterStage::AUTO); - filter.priorFilter(gv); - } - } else { - qcmanager_->priorFilter(gv); // No operation, placeholder - for (ObsFilter& filter : priorFilters_) { - filter.checkFilterData(FilterStage::PRIOR); - filter.priorFilter(gv); - } +void ObsFilters::priorFilter(const GeoVaLs & gv) { + if (autoFilters_.size() > 0) { + for (ObsFilter & filter : autoFilters_) { + filter.checkFilterData(FilterStage::AUTO); + filter.priorFilter(gv); + } + } else { + for (ObsFilter & filter : priorFilters_) { + filter.checkFilterData(FilterStage::PRIOR); + filter.priorFilter(gv); } } } // ----------------------------------------------------------------------------- -void ObsFilters::postFilter(const GeoVaLs& gv, const ioda::ObsVector& hofx, - const ioda::ObsVector& bias, - const ObsDiagnostics& diags) { - if (atLeastOneFilterConfigured_) { - if (autoFilters_.size() > 0) { - qcmanager_->postFilter(gv, hofx, bias, diags); - for (ObsFilter& filter : autoFilters_) { - filter.checkFilterData(FilterStage::AUTO); - filter.postFilter(gv, hofx, bias, diags); - } - } else { - qcmanager_->postFilter(gv, hofx, bias, diags); - for (ObsFilter& filter : postFilters_) { - filter.checkFilterData(FilterStage::POST); - filter.postFilter(gv, hofx, bias, diags); - } +void ObsFilters::postFilter(const GeoVaLs & gv, + const ioda::ObsVector & hofx, + const ioda::ObsVector & bias, + const ObsDiagnostics & diags) { + if (autoFilters_.size() > 0) { + for (ObsFilter & filter : autoFilters_) { + filter.checkFilterData(FilterStage::AUTO); + filter.postFilter(gv, hofx, bias, diags); + } + } else { + for (ObsFilter & filter : postFilters_) { + filter.checkFilterData(FilterStage::POST); + filter.postFilter(gv, hofx, bias, diags); } } } // ----------------------------------------------------------------------------- -void ObsFilters::print(std::ostream& os) const { +void ObsFilters::print(std::ostream & os) const { if (autoFilters_.size() > 0) { os << "ObsFilters: " << autoFilters_.size() << " elements:" << std::endl; - for (const ObsFilter& filter : autoFilters_) os << filter << std::endl; + for (const ObsFilter & filter : autoFilters_) + os << filter << std::endl; } else { os << "Pre filters: " << preFilters_.size() << " elements:" << std::endl; - for (const ObsFilter& preFilter : preFilters_) os << preFilter << std::endl; + for (const ObsFilter & preFilter : preFilters_) + os << preFilter << std::endl; os << "Prior filters: " << priorFilters_.size() << " elements:" << std::endl; - for (const ObsFilter& priorFilter : priorFilters_) os << priorFilter << std::endl; + for (const ObsFilter & priorFilter : priorFilters_) + os << priorFilter << std::endl; os << "Post filters: " << postFilters_.size() << " elements:" << std::endl; - for (const ObsFilter& postFilter : postFilters_) os << postFilter << std::endl; + for (const ObsFilter & postFilter : postFilters_) + os << postFilter << std::endl; } } diff --git a/src/ufo/ObsFilters.h b/src/ufo/ObsFilters.h index 35e48e6a6..c7d8db939 100644 --- a/src/ufo/ObsFilters.h +++ b/src/ufo/ObsFilters.h @@ -12,7 +12,6 @@ #include -#include "ioda/ObsDataVector.h" #include "oops/base/ObsVariables.h" #include "oops/base/Variables.h" @@ -66,6 +65,7 @@ class ObsFiltersParameters : public oops::Parameters { class ObsFilters : public util::Printable, private boost::noncopyable { typedef std::vector FilterParams_; + template using ObsDataPtr_ = std::shared_ptr >; public: /// Initialize all filters for \p obspace, from parameters, using @@ -74,7 +74,7 @@ class ObsFilters : public util::Printable, /// assimilation. ObsFilters(ioda::ObsSpace &, const eckit::Configuration &, - ioda::ObsDataVector & flags, ioda::ObsDataVector & obserr, + ObsDataPtr_ qcflags, ObsDataPtr_ obserr, const int iteration = 0); void preProcess(); @@ -95,9 +95,6 @@ class ObsFilters : public util::Printable, std::vector & filters); ioda::ObsSpace & obsspace_; - /// QCmanager filter to handle QC flags and collect statistics, - /// whenever at least one filter is configured. - std::unique_ptr qcmanager_; // List of filters for which the stage (pre/prior/post) will be determined automatically. std::vector autoFilters_; // List of filters which have been designated to run at the pre stage. @@ -108,9 +105,8 @@ class ObsFilters : public util::Printable, std::vector postFilters_; oops::Variables geovars_; oops::ObsVariables diagvars_; - ioda::ObsDataVector & qcflags_; - ioda::ObsDataVector & obserr_; - bool atLeastOneFilterConfigured_ = false; + ObsDataPtr_ qcflags_; + ObsDataPtr_ obserrfilter_; const int iteration_; }; diff --git a/src/ufo/ObsTraits.h b/src/ufo/ObsTraits.h index 84c2ab857..9b66e4a08 100644 --- a/src/ufo/ObsTraits.h +++ b/src/ufo/ObsTraits.h @@ -1,8 +1,8 @@ /* * (C) Copyright 2017-2020 UCAR - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ #ifndef UFO_OBSTRAITS_H_ @@ -11,7 +11,6 @@ #include #include "ioda/ObsDataVector.h" -#include "ioda/ObsIterator.h" #include "ioda/ObsSpace.h" #include "ioda/ObsVector.h" @@ -40,7 +39,6 @@ struct ObsTraits { template using ObsDataVector = ioda::ObsDataVector; typedef ufo::ObsError ObsError; typedef ufo::ObsFilters ObsFilter; - typedef ioda::ObsIterator GeometryIterator; typedef ufo::ObsOperator ObsOperator; typedef ufo::LinearObsOperator LinearObsOperator; diff --git a/src/ufo/errors/ObsErrorBase.h b/src/ufo/errors/ObsErrorBase.h index b08619b33..531649d3b 100644 --- a/src/ufo/errors/ObsErrorBase.h +++ b/src/ufo/errors/ObsErrorBase.h @@ -93,8 +93,6 @@ class ObsErrorParametersWrapper : public oops::Parameters { /// is deserialized. oops::PolymorphicParameter errorParameters{"covariance model", "diagonal", this}; - - const ObsErrorParametersBase & errorParams() const {return this->errorParameters;} }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/errors/ObsErrorCrossVarCov.cc b/src/ufo/errors/ObsErrorCrossVarCov.cc index f24eaece1..a97126f3c 100644 --- a/src/ufo/errors/ObsErrorCrossVarCov.cc +++ b/src/ufo/errors/ObsErrorCrossVarCov.cc @@ -34,7 +34,7 @@ ObsErrorCrossVarCov::ObsErrorCrossVarCov(const Parameters_ & params, varcorrelations_(Eigen::MatrixXd::Identity(stddev_.nvars(), stddev_.nvars())), reconditioner_(nullptr) { - oops::Log::trace() << "ObsErrorCrossVarCov::ObsErrorCrossVarCov starting" << std::endl; + // deserialize configuration into ObsErrorCrossVarCovParameters // Create reconditioner reconditioner_.reset(new ObsErrorReconditioner(params_.reconditioning.value())); // Open and read error correlations from the hdf5 file @@ -107,7 +107,6 @@ ObsErrorCrossVarCov::ObsErrorCrossVarCov(const Parameters_ & params, << " channels/variables in file: " << params.inputFile.value() << ". To see which channels, turn on OOPS_TRACE\n" << std::endl; } - oops::Log::trace() << "ObsErrorCrossVarCov::ObsErrorCrossVarCov finished" << std::endl; } // ----------------------------------------------------------------------------- @@ -310,80 +309,22 @@ void ObsErrorCrossVarCov::inverseMultiply(ioda::ObsVector & dy) const { // ----------------------------------------------------------------------------- void ObsErrorCrossVarCov::localize(ioda::ObsVector & locvector) const { - oops::Log::trace() << "ufo::ObsErrorCrossVarCov::localize start" << std::endl; - - const double missing = util::missingValue(); - - ASSERT_MSG(locvector.size() == stddev_.size(), - "ObsErrorCrossVarCov::localize: " - "Localization vector dimension must match dimension of obs space."); - std::vector localstddev; - const size_t nlocs = locvector.nlocs(); - const size_t nvars = locvector.nvars(); - local_jvars_.clear(); - local_nobs_.clear(); - for (size_t jloc = 0; jloc < nlocs; ++jloc) { - size_t nused = 0; - for (size_t jvar = 0; jvar < nvars; ++jvar) { - size_t jj = jloc*nvars + jvar; - if (locvector[jj] != missing && locvector[jj] <= 0) { - throw eckit::BadValue("Localization weights must be positive. Use " - "oops::util::missingValue() to indicate " - "an observation with a weight of zero."); - } - if (locvector[jj] != missing && stddev_[jj] != missing) { - localstddev.push_back(stddev_[jj] * std::pow(locvector[jj], -0.5)); - local_jvars_.push_back(jvar); - ++nused; - } - } - if (nused > 0) local_nobs_.push_back(nused); - } - local_stddev_ = Eigen::Map(localstddev.data(), localstddev.size()); + throw eckit::BadParameter("Trying to localize a correlated R matrix, this is " + "not yet implemented."); } int ObsErrorCrossVarCov::localDim() const { - return local_stddev_.size(); + throw eckit::BadParameter("Trying to localize a correlated R matrix, this is " + "not yet implemented."); } Eigen::MatrixXf ObsErrorCrossVarCov::localInverseMultiply(const Eigen::MatrixXf & zz) const { - ASSERT_MSG(zz.cols() == localDim(), - "ObsErrorCrossVarCov::localInverseMultiply: " - "Input vector dimension must match local R matrix dimension"); - Eigen::MatrixXf zzRinv(zz.rows(), zz.cols()); - size_t j_obs = 0; - // loop over obs locations in local volume - for (int pt_nobs : local_nobs_) { - // construct pt std dev and correlations for obs at this point - Eigen::VectorXf pt_stddev = local_stddev_.segment(j_obs, pt_nobs).cast(); - Eigen::MatrixXf pt_corr = Eigen::MatrixXf::Identity(pt_nobs, pt_nobs); - for (size_t ipt = 0; ipt < pt_nobs; ++ipt) { - for (size_t jpt = 0; jpt < ipt; ++jpt) { - // only fill lower triangle as that's all we need for LLT - pt_corr(ipt, jpt) = static_cast(varcorrelations_(local_jvars_[j_obs+ipt], - local_jvars_[j_obs+jpt])); - } - } - // Cholesky decomposition to allow inverse multiplication - Eigen::LLT LLT(pt_corr); - for (size_t ii = 0; ii < zz.rows(); ++ii) { - Eigen::VectorXf row = zz(ii, Eigen::seq(j_obs, j_obs+pt_nobs-1)).transpose(); - // D^{-1/2} * row - row.array() /= pt_stddev.array(); - // C^{-1} * D^{-1/2} * row - LLT.solveInPlace(row); - // D^{-1/2} * C^{-1} * D^{-1/2} * row - row.array() /= pt_stddev.array(); - // store in output - zzRinv(ii, Eigen::seq(j_obs, j_obs+pt_nobs-1)) = row.transpose(); - } - j_obs += pt_nobs; - } - return zzRinv; + throw eckit::BadParameter("Localisation not implemented for correlated R matrices"); } Eigen::VectorXd ObsErrorCrossVarCov::local_invVarR() const { - throw eckit::BadParameter("ObsErrorCrossVarCov::local_invVarR not implemented."); + throw eckit::BadParameter("Local inverse variance not implemented for " + "correlated R matrices"); } // ----------------------------------------------------------------------------- diff --git a/src/ufo/errors/ObsErrorCrossVarCov.h b/src/ufo/errors/ObsErrorCrossVarCov.h index a7c8c5663..4d18033e8 100644 --- a/src/ufo/errors/ObsErrorCrossVarCov.h +++ b/src/ufo/errors/ObsErrorCrossVarCov.h @@ -13,7 +13,6 @@ #include #include -#include #include "ioda/ObsVector.h" @@ -39,6 +38,9 @@ class ObsErrorCrossVarCovParameters : public ObsErrorParametersBase { /// Input file containing correlations or covariances. If covariances are /// specified, they will be converted to correlations. oops::RequiredParameter inputFile{"input file", this}; + + oops::Parameter reconditioning{"reconditioning", + ObsErrorReconditionerParameters(), this}; }; // ----------------------------------------------------------------------------- /// \brief Observation error covariance matrix with cross-variable @@ -110,12 +112,6 @@ class ObsErrorCrossVarCov : public ObsErrorBase { Parameters_ params_; /// Observation error standard deviations ioda::ObsVector stddev_; - /// Localised observation error standard deviations - mutable Eigen::VectorXd local_stddev_; - /// Variable indices of local obs, used to construct local correlations - mutable std::vector local_jvars_; - /// Number of local obs at each location, used to construct local correlations - mutable std::vector local_nobs_; /// Variables for which correlations are defined (same as ObsSpace::obsvariables()) const oops::ObsVariables vars_; /// Correlations between variables diff --git a/src/ufo/errors/ObsErrorDiagonal.cc b/src/ufo/errors/ObsErrorDiagonal.cc index 86906d993..429f1dad5 100644 --- a/src/ufo/errors/ObsErrorDiagonal.cc +++ b/src/ufo/errors/ObsErrorDiagonal.cc @@ -92,7 +92,7 @@ void ObsErrorDiagonal::localize(ioda::ObsVector & locvector) const { Eigen::MatrixXf ObsErrorDiagonal::localInverseMultiply(const Eigen::MatrixXf & zz) const { Eigen::MatrixXf zzRinv(zz.rows(), zz.cols()); for (int ii = 0; ii < zz.rows(); ++ii) { - zzRinv(ii, Eigen::placeholders::all) = zz(ii, Eigen::placeholders::all) + zzRinv(ii, Eigen::all) = zz(ii, Eigen::all) .cwiseProduct(local_inverseVariance_.cast().transpose()); } return zzRinv; diff --git a/src/ufo/errors/ObsErrorDiagonalInvGamma.cc b/src/ufo/errors/ObsErrorDiagonalInvGamma.cc index 77da67f7f..7ff75d25e 100644 --- a/src/ufo/errors/ObsErrorDiagonalInvGamma.cc +++ b/src/ufo/errors/ObsErrorDiagonalInvGamma.cc @@ -94,7 +94,7 @@ void ObsErrorDiagonalInvGamma::localize(ioda::ObsVector & locvector) const { Eigen::MatrixXf ObsErrorDiagonalInvGamma::localInverseMultiply(const Eigen::MatrixXf & zz) const { Eigen::MatrixXf zzRinv(zz.rows(), zz.cols()); for (int ii = 0; ii < zz.rows(); ++ii) { - zzRinv(ii, Eigen::placeholders::all) = zz(ii, Eigen::placeholders::all) + zzRinv(ii, Eigen::all) = zz(ii, Eigen::all) .cwiseProduct(local_inverseVariance_.cast().transpose()); } return zzRinv; diff --git a/src/ufo/errors/ObsErrorDiffusion.cc b/src/ufo/errors/ObsErrorDiffusion.cc index 7dcc65afc..4e9a5ebeb 100644 --- a/src/ufo/errors/ObsErrorDiffusion.cc +++ b/src/ufo/errors/ObsErrorDiffusion.cc @@ -10,12 +10,9 @@ #include #include -#include "atlas/grid/Grid.h" #include "atlas/mesh/Mesh.h" #include "atlas/mesh/MeshBuilder.h" #include "atlas/output/Gmsh.h" -#include "atlas/util/Geometry.h" -#include "atlas/util/KDTree.h" #include "eckit/exception/Exceptions.h" #include "eckit/mpi/Comm.h" @@ -50,25 +47,23 @@ ObsErrorDiffusion::ObsErrorDiffusion(const Parameters_ & params, inverseVariance_ = stddev_; inverseVariance_ *= stddev_; inverseVariance_.invert(); - oops::Log::trace() << "ObsErrorDiffusion::ObsErrorDiffusion constructor start" << std::endl; + oops::Log::trace() << "ObsErrorDiffusion:ObsErrorDiffusion constructed nobs = " + << stddev_.nobs() << std::endl; // Set up diffusion grid as atlas functionspace - const atlas::idx_t nlocs = stddev_.nlocs(); + size_t nlocs = stddev_.nlocs(); std::vector lons(nlocs); std::vector lats(nlocs); std::vector gridXY(2*nlocs); - std::vector obsnodes(nlocs); // Get obs locations from obs space odb.get_db("MetaData", "longitude", lons); odb.get_db("MetaData", "latitude", lats); // fill vector in form: [ lon_0, lat_0, lon_1, lat_1, ... ] - for (atlas::idx_t i = 0; i < nlocs; ++i) { + for (size_t i = 0; i < nlocs; ++i) { gridXY[2*i] = lons[i]; gridXY[2*i+1] = lats[i]; - obsnodes[i] = atlas::PointLonLat(lons[i], lats[i]); - obsnodes[i].normalise(); } eckit::LocalConfiguration fspaceConfig; @@ -84,64 +79,22 @@ ObsErrorDiffusion::ObsErrorDiffusion(const Parameters_ & params, atlas::Mesh mesh{}; atlas::FieldSet fset{}; -//------------------------------------------------------------------------------ -// If control grid parameters are provided, -// use them to create a coarser grid for the diffusion operator -// -// NOTE: not setting control grid as default as the grid spacing and -// remove within parameters should be explicitly set, -// depending on the observation network geometry, -// to ensure the control grid is created as intended -//------------------------------------------------------------------------------ - if (params_.controlGrid.value()) { - const int gridSpacing = params_.controlGrid.value()->gridSpacing.value(); - const double removeWithin = params_.controlGrid.value()->removeWithin.value(); - - // returns merged obs + remaining control grid points - std::vector mergedPoints = - createControlGrid(obsnodes, removeWithin, gridSpacing); - // store offset before updating gridXY - const atlas::idx_t nMerged = mergedPoints.size(); - obsOffset_ = nMerged - nlocs; - - // update gridXY with merged points for mesh creation - gridXY.resize(2 * mergedPoints.size()); - for (atlas::idx_t k = 0; k < nMerged; k++) { - gridXY[2*k] = mergedPoints[k].lon(); - gridXY[2*k+1] = mergedPoints[k].lat(); - } - fspaceConfig.set("grid.xy", gridXY); - } // Uses Delauney triangulation for mesh generation util::setupFunctionSpace(comm_, fspaceConfig, grid, partitioner, mesh, fspace, fset); - geom_.reset(new oops::GeometryData(fspace, fset, true, comm_)); - diffusionGeom_ = oops::Diffusion::calculateDerivedGeom(std::as_const(*geom_)); - diffusion_.reset(new oops::Diffusion(std::as_const(*geom_), diffusionGeom_)); - - const atlas::idx_t npts = geom_->functionSpace().size(); - // Save mesh and point-type marker field for visualization with gmsh + // save output for viewing with gmsh if (params_.outputDiffusionMesh.value()) { - // Create marker field: 0 = control grid point, 1 = obs point - atlas::Field markerField = geom_->functionSpace().createField( - atlas::option::levels(1) | atlas::option::name("pointType")); - auto v_marker = atlas::array::make_view(markerField); - v_marker.assign(0.0); - for (atlas::idx_t i = 0; i < nlocs; ++i) { - v_marker(obsOffset_ + i, 0) = 1.0; // set marker for obspoints to 1 - } - fset.add(markerField); // Add to fieldset before writing - // Write mesh with the marker field - const std::string filename = "diffusion_mesh.msh"; + std::string filename = "diffusion_mesh.msh"; atlas::output::Gmsh gmsh(filename, atlas::util::Config("coordinates", "xyz") | atlas::util::Config("ghost", true)); // enables viewing halos per task gmsh.write(mesh); - gmsh.write(fset, geom_->functionSpace()); - oops::Log::info() << "ObsErrorDiffusion: Saved diffusion mesh to " - << filename << std::endl; } + geom_.reset(new oops::GeometryData(fspace, fset, true, comm_)); + diffusionGeom_ = oops::Diffusion::calculateDerivedGeom(std::as_const(*geom_)); + diffusion_.reset(new oops::Diffusion(std::as_const(*geom_), diffusionGeom_)); + // ------------------------------------------------------------ // Create horizontal length scales for normalization // ------------------------------------------------------------ @@ -151,10 +104,6 @@ ObsErrorDiffusion::ObsErrorDiffusion(const Parameters_ & params, auto v_hzScales = atlas::array::make_view(hzScales); const double val = params_.lscale; v_hzScales.assign(val); - // Set the scales for control points to 0 so they don't influence the diffusion of obs points - for (atlas::idx_t i = 0; i < obsOffset_; ++i) { - v_hzScales(i, 0) = 0; - } atlas::FieldSet scales; scales.add(hzScales); @@ -207,7 +156,7 @@ ObsErrorDiffusion::ObsErrorDiffusion(const Parameters_ & params, // ---- Update Welford running variance ---- auto v_rand = atlas::array::make_view(randSet["rand"]); - for (atlas::idx_t i = 0; i < npts; i++) { + for (atlas::idx_t i = 0; i < nlocs; i++) { double f = v_rand(i, 0); double old_m = v_m(i, 0); double new_m = old_m + (f - old_m) / itr; @@ -217,14 +166,12 @@ ObsErrorDiffusion::ObsErrorDiffusion(const Parameters_ & params, } // ---- Finalize normalization: Γ_i = 1/sqrt(Var_i) ---- - for (atlas::idx_t i = 0; i < npts; i++) { + for (atlas::idx_t i = 0; i < nlocs; i++) { if (v_s(i, 0) > 0.0) { v_norm(i, 0) = 1.0 / std::sqrt(v_s(i, 0) / (randomizationIterations - 1)); } } - oops::Log::trace() << "ObsErrorDiffusion::ObsErrorDiffusion constructed nobs = " - << stddev_.nobs() << std::endl; -} // end constructor +} // ----------------------------------------------------------------------------- @@ -261,8 +208,7 @@ void ObsErrorDiffusion::multiply(ioda::ObsVector & dy) const { // NOTE: This will only work for reading a single variable // (with single channel) from the obsSpace - const atlas::idx_t nlocs = dy.nlocs(); - const atlas::idx_t npts = geom_->functionSpace().size(); + size_t nlocs = dy.nlocs(); atlas::FieldSet fset; for (std::string var : params_.var.value()) { @@ -271,8 +217,8 @@ void ObsErrorDiffusion::multiply(ioda::ObsVector & dy) const { // copy data from obsVector to field auto obsView = atlas::array::make_view(obs); - for (atlas::idx_t i = 0; i < nlocs; ++i) { - obsView(obsOffset_ + i, 0) = dy.data()[i]; + for (atlas::idx_t i = 0; i < obsView.shape(0); ++i) { + obsView(i, 0) = dy.data()[i]; } // add obs "var" to fieldSet @@ -290,7 +236,7 @@ void ObsErrorDiffusion::multiply(ioda::ObsVector & dy) const { auto applyNormSqrt = [&](atlas::Field & f) { auto v_f = atlas::array::make_view(f); auto v_norm = atlas::array::make_view(this->hzNorm_); - for (atlas::idx_t i = 0; i < npts; i++) { + for (atlas::idx_t i = 0; i < nlocs; i++) { v_f(i, 0) *= v_norm(i, 0); } }; @@ -307,9 +253,8 @@ void ObsErrorDiffusion::multiply(ioda::ObsVector & dy) const { // move this step into diffusion loop (above) for multiple variables? for (auto field : fset) { auto fieldView = atlas::array::make_view(field); - // using nlocs here since only copying obs points back to dy - for (atlas::idx_t i = 0; i < nlocs; ++i) { - dy[i] = fieldView(obsOffset_ + i, 0); + for (atlas::idx_t i = 0; i < fieldView.shape(0); ++i) { + dy[i] = fieldView(i, 0); } } @@ -367,8 +312,8 @@ void ObsErrorDiffusion::randomize(ioda::ObsVector & dy) const { // NOTE:: double check this works for list of more than one var in yaml for (std::string var : vars) { auto randView = atlas::array::make_view(rand[var]); - for (atlas::idx_t i = 0; i < dy.nlocs(); ++i) { - dy[i] = randView(obsOffset_ + i, 0); + for (int i = 0; i < randView.shape(0); ++i) { + dy[i] = randView(i, 1); } } } @@ -426,69 +371,5 @@ void ObsErrorDiffusion::print(std::ostream & os) const { // ----------------------------------------------------------------------------- -std::vector ObsErrorDiffusion::createControlGrid( - const std::vector& obsnodes, - const double removeWithin, - const int gridSpacing) - { - oops::Log::trace() << "ObsErrorDiffusion CreateControlGrid: start " << std::endl; - const atlas::idx_t nlocs = obsnodes.size(); - // --- 1. Build control grid --- - // -------------------------------- - // number of grid points in longitude - const int nx = static_cast(std::round(360.0 / gridSpacing)); - // number of grid points in latitude with equator in center - const int ny = static_cast(std::round(180.0 / gridSpacing)) + 1; - const atlas::util::Config cfg = atlas::util::Config("type", "regular_lonlat") - ("nx", nx) - ("ny", ny) - ("lon0", 0.0) - ("lat0", -90.0); - atlas::RegularLonLatGrid cntrlGrid(cfg); - // --- 2. Build KD-tree from obs --- - // ------------------------------------- - atlas::util::IndexKDTree obstree; // declare kd tree - std::vector indices(nlocs); // for payload - std::iota(indices.begin(), indices.end(), 0); // assign index to obs payload - obstree.build(obsnodes, indices); // build KDTree from obs locations - // --- 3. Filter control grid points --- - // -------------------------------------- - std::vector remainingGridPoints; - remainingGridPoints.reserve(nx * ny); - int nRemoved = 0; - for (atlas::idx_t j = 0; j < cntrlGrid.ny(); j++) { - for (atlas::idx_t i = 0; i < cntrlGrid.nx(); i++) { - const atlas::PointLonLat gridPt = cntrlGrid.lonlat(i, j); - const auto result = obstree.closestPoint(gridPt); - if (result.distance() <= removeWithin) { - nRemoved++; - oops::Log::debug() << "ObsErrorDiffusion: CreateControlGrid:" - << "Removed grid point (" - << gridPt.lon() << ", " << gridPt.lat() - << ") — nearest obs #" << result.payload() - << " (lon=" << obsnodes[result.payload()].lon() - << ", lat=" << obsnodes[result.payload()].lat() - << ") at distance=" << result.distance() << "m\n"; - } else { - remainingGridPoints.push_back(gridPt); - } - } - } - oops::Log::info() << "ObsErrorDiffusion: CreateControlGrid:" - << "Control grid total=" - << nx*ny - << " removed=" << nRemoved - << " remaining=" << remainingGridPoints.size() << std::endl; - // --- 4. Merge remaining control grid points with obs locations --- - //--------------------------------------------------------------------- - std::vector mergedPoints; - mergedPoints.reserve(remainingGridPoints.size() + nlocs); - mergedPoints.insert(mergedPoints.end(), - remainingGridPoints.begin(), remainingGridPoints.end()); - mergedPoints.insert(mergedPoints.end(), - obsnodes.begin(), obsnodes.end()); - oops::Log::trace() << "ObsErrorDiffusion: CreateControlGrid: end" << std::endl; - return mergedPoints; - } } // namespace ufo diff --git a/src/ufo/errors/ObsErrorDiffusion.h b/src/ufo/errors/ObsErrorDiffusion.h index ce36cedb5..ffa604092 100644 --- a/src/ufo/errors/ObsErrorDiffusion.h +++ b/src/ufo/errors/ObsErrorDiffusion.h @@ -37,13 +37,6 @@ namespace ufo { class ObsErrorDiffusionParameters : public ObsErrorParametersBase { OOPS_CONCRETE_PARAMETERS(ObsErrorDiffusionParameters, ObsErrorParametersBase) public: - class ControlGrid : public oops::Parameters { - OOPS_CONCRETE_PARAMETERS(ControlGrid, oops::Parameters) - public: - oops::RequiredParameter gridSpacing{"grid spacing in degrees", this}; - oops::RequiredParameter removeWithin{"remove within meters", this}; - }; - oops::RequiredParameter> var{"correlation variable names", "Group/Name obs variable that correlations should be computed for (note: " "this variable should be the same variable as obs space is grouped on", this}; @@ -54,11 +47,8 @@ class ObsErrorDiffusionParameters : public ObsErrorParametersBase { 100, this}; oops::Parameter InverseAccuracy{"Accuracy of inverse", 1.0e-7, this}; oops::Parameter outputDiffusionMesh{"Output diffusion mesh", false, this}; - // Add parameter for control grid - oops::OptionalParameter controlGrid{"control grid", this}; }; - // ----------------------------------------------------------------------------- /// \brief Correlated observation error covariance matrix. class ObsErrorDiffusion : public ObsErrorBase { @@ -137,17 +127,6 @@ class ObsErrorDiffusion : public ObsErrorBase { const eckit::mpi::Comm & comm_; std::unique_ptr diffusion_; atlas::Field hzNorm_; - - // Offset where obs locations start in the merged function space - // (0 if no control grid, nRemainingControlPoints otherwise) - atlas::idx_t obsOffset_ = 0; // <-- add here - - /// Creates control grid, removes close control points and - // returns merged grid points (remaining control grid + obs locations) - std::vector createControlGrid( - const std::vector& obsnodes, - const double removeWithin, - const int gridSpacing); }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/errors/ObsErrorParametersBase.h b/src/ufo/errors/ObsErrorParametersBase.h index c3ad40ba7..e493292d2 100644 --- a/src/ufo/errors/ObsErrorParametersBase.h +++ b/src/ufo/errors/ObsErrorParametersBase.h @@ -14,8 +14,6 @@ #include "oops/util/parameters/Parameter.h" #include "oops/util/parameters/Parameters.h" -#include "ufo/errors/ObsErrorReconditioner.h" - namespace ufo { /// \brief Base obs errors parameters class @@ -26,9 +24,6 @@ class ObsErrorParametersBase : public oops::Parameters { oops::Parameter model{"covariance model", "diagonal", this}; oops::Parameter RMSEtolerance{"Obs Error test tolerance", "RMSE tolerance for" "oops::ObsErrorCovariance test", 1.0e-10, this}; - - oops::Parameter reconditioning{"reconditioning", - ObsErrorReconditionerParameters(), this}; }; } // namespace ufo diff --git a/src/ufo/errors/ObsErrorReconditioner.cc b/src/ufo/errors/ObsErrorReconditioner.cc index 93b5f8203..090c5c6af 100644 --- a/src/ufo/errors/ObsErrorReconditioner.cc +++ b/src/ufo/errors/ObsErrorReconditioner.cc @@ -16,7 +16,6 @@ ObsErrorReconditionerMethodParameterTraitsHelper::namedValues[]; ObsErrorReconditioner::ObsErrorReconditioner(const Parameters_ & params) : params_(params) { - oops::Log::trace() << "ObsErrorReconditioner::ObsErrorReconditioner starting" << std::endl; // Checking valid reconditioning options if reconditioning specified. ufo::ObsErrorReconditionerMethod recon_method = params_.ReconMethod.value(); size_t nvalid_options = 0; @@ -45,12 +44,10 @@ ObsErrorReconditioner::ObsErrorReconditioner(const Parameters_ & params) break; case ufo::ObsErrorReconditionerMethod::NORECONDITIONING: oops::Log::trace() << "'No reconditioning' option selected, " - "for advanced reconditioning; " "recondition method can be tested, " "R matrix should not change.\n"; break; } - oops::Log::trace() << "ObsErrorReconditioner::ObsErrorReconditioner finished" << std::endl; } // ObsErrorReconditioner::ObsErrorReconditioner void ObsErrorReconditioner::recondition(Eigen::MatrixXd & R) const { @@ -64,7 +61,8 @@ void ObsErrorReconditioner::recondition(Eigen::MatrixXd & R) const { // Check square matrix size_t nrows = R.rows(); - assert(nrows == R.cols()); + size_t ncols = R.cols(); + assert(nrows == ncols); // Performing eigendecomposition oops::Log::trace() << "R before reconditioning:\n" << R << std::endl << std::endl; diff --git a/src/ufo/errors/ObsErrorWithinGroupCov.cc b/src/ufo/errors/ObsErrorWithinGroupCov.cc index 19cf9512c..ff034d312 100644 --- a/src/ufo/errors/ObsErrorWithinGroupCov.cc +++ b/src/ufo/errors/ObsErrorWithinGroupCov.cc @@ -60,19 +60,6 @@ double markov(const double & distnorm, const double & maxnormdist) { return markovvalue; } -double gaussian(const double & distnorm) { - // computes a Gaussian-shaped localization -/* - // cutoff to zero at normalized distance of 2.5 - double gaussVal = 0.0; - if (distnorm < 2.5) { - gaussVal = exp(-0.5*pow(distnorm, 2)); - } - return gaussVal; -*/ - return exp(-0.5*pow(distnorm, 2)); -} - // ----------------------------------------------------------------------------- // Distance functions - base class and linear distance // ----------------------------------------------------------------------------- @@ -118,15 +105,11 @@ ObsErrorWithinGroupCov::ObsErrorWithinGroupCov(const Parameters_ & params, params_(params), obspace_(obspace), coord_(coord_constructor(params_, obspace)), - stddev_(obspace, "ObsError"), - reconditioner_(nullptr) + stddev_(obspace, "ObsError") { correlations_.reserve(obspace.nrecs()); double missing_double = util::missingValue(); - // Create reconditioner - reconditioner_.reset(new ObsErrorReconditioner(params_.reconditioning.value())); - for (auto irec = obspace.recidx_begin(); irec != obspace.recidx_end(); ++irec) { std::vector rec_idx = obspace.recidx_vector(irec); size_t rec_nobs = rec_idx.size(); @@ -161,22 +144,14 @@ ObsErrorWithinGroupCov::ObsErrorWithinGroupCov(const Parameters_ & params, corr(jloc, iloc) = markov(distance / params_.lscale.value(), params_.markovLengthscaleFactor.value()); break; - case CorrelationFunctions::GAUSSIAN: - corr(jloc, iloc) = gaussian(distance / params_.lscale.value()); - break; } } } - // The Markov and Gaussian correlation profiles do not guarantee a positive definite matrix - // therefore a check is needed. If the matrix is not positive definite the diagonal is - // slightly inflated to make it positive definite which has the effect of slightly inflating - // the observation errors. - if (params_.correlationFunction.value() == CorrelationFunctions::MARKOV || - (params_.correlationFunction.value() == CorrelationFunctions::GAUSSIAN && - params_.applyBasicReconditioning.value())) { - oops::Log::trace() << "ObsErrorWithinGroupCov::ObsErrorWithinGroupCov " - << "basic reconditining enabled. " - << "Checking eigenvalues of correlation matrix" << std::endl; + // The Markov correlation method does not guarantee a positive definite matrix therefore a + // check is needed. If the matrix is not positive definite the diagonal is slightly inflated + // to make it positive definite which has the effect of slightly inflating the observation + // errors. + if (params_.correlationFunction.value() == CorrelationFunctions::MARKOV) { Eigen::SelfAdjointView corr_view(corr); Eigen::SelfAdjointEigenSolver es(corr_view); double min_eigenvalue = es.eigenvalues().minCoeff(); @@ -189,119 +164,17 @@ ObsErrorWithinGroupCov::ObsErrorWithinGroupCov(const Parameters_ & params, } } correlations_.push_back(corr); - } // end loop over records + } } // ----------------------------------------------------------------------------- void ObsErrorWithinGroupCov::update(const ioda::ObsVector & obserr) { - oops::Log::trace() << "ObsErrorWithinGroupCov::update() start" << std::endl; stddev_ = obserr; - if (params_.reconditioning.value().ReconMethod.value() != - ufo::ObsErrorReconditionerMethod::NORECONDITIONING) { - this->recondition(obserr); - } - oops::Log::trace() << "ObsErrorWithinGroupCov::update() end" << std::endl; } // ----------------------------------------------------------------------------- -void ObsErrorWithinGroupCov::recondition(const ioda::ObsVector & mask) { - oops::Log::trace() << "ObsErrorWithinGroupCov::recondition() start" << std::endl; - const size_t nlocs = mask.nlocs(); - const size_t nvars = mask.nvars(); - const double missing = util::missingValue(); - - for (Eigen::MatrixXd groupCorr : correlations_) { - // preallocate data - Eigen::MatrixXd avgcorr = Eigen::MatrixXd::Zero(groupCorr.rows(), groupCorr.cols()); - size_t nused_locs = 0; - const double dnlocs = static_cast(nlocs); - - // Masking and packing R matrix at each location - // loop over all observations locations - for (size_t jloc = 0; jloc < nlocs; ++jloc) { - std::vector usedobs_indices(nvars); - // Calculating variables that are used from mask - // These are the variables that pass QC. - size_t nused = 0; - for (size_t jvar = 0; jvar < nvars; ++jvar) - if (mask[jloc * nvars + jvar] != missing) usedobs_indices[nused++] = jvar; - - // Initialising correlation matrix for the given location. - // This will be a fraction of the correlation matrix after - // the reconditioning has happened. - Eigen::MatrixXd corr_at_loc(groupCorr / dnlocs); - if (nused <= 1) { - oops::Log::trace() << "nused = " << nused - << "at jloc = " << jloc - << ", skipping reconditioning.\n"; - continue; - } - oops::Log::trace() << "\nReconditioning R matrix at jloc = " << jloc << std::endl; - Eigen::MatrixXd R = Eigen::MatrixXd::Zero(nused, nused); - - // loop over all used variables at location - for (size_t jvar = 0; jvar < nused; ++jvar) { - const size_t ivar = usedobs_indices[jvar]; - const size_t ind = jloc * nvars + ivar; - R(jvar, jvar) = stddev_[ind]*stddev_[ind]; - for (size_t jvar2 = jvar + 1; jvar2 < nused; ++jvar2) { - const size_t ivar2 = usedobs_indices[jvar2]; - const size_t ind2 = jloc * nvars + ivar2; - R(jvar, jvar2) = groupCorr(ivar, ivar2) - * stddev_[ind] - * stddev_[ind2]; - R(jvar2, jvar) = groupCorr(ivar2, ivar) - * stddev_[ind2] - * stddev_[ind]; - } - } - - // Recondition the R matrix - this->reconditioner_->recondition(R); - - // Unpacking the reconditioned matrix - // into stddev_ and groupCorr members - // loop over all used variables at location - for (size_t jvar = 0; jvar < nused; ++jvar) { - const size_t ivar = usedobs_indices[jvar]; - const size_t ind = jloc * nvars + ivar; - // Updating stddev_ - const double stddev_ind = std::sqrt(R(jvar, jvar)); - stddev_[ind] = stddev_ind; - for (size_t jvar2 = jvar + 1; jvar2 < nused; ++jvar2) { - // Ensuring location independent correlations - // by using average of reconditioned correlations at each location - const double stddev_ind2 = std::sqrt(R(jvar2, jvar2)); - const size_t ivar2 = usedobs_indices[jvar2]; - corr_at_loc(ivar, ivar2) = R(jvar, jvar2) - / (stddev_ind - * stddev_ind2 - * dnlocs); - corr_at_loc(ivar2, ivar) = R(jvar2, jvar) - / (stddev_ind2 - * stddev_ind - * dnlocs); - } - } - nused_locs++; - avgcorr += corr_at_loc; - } - - // Reassigning the correlations to the renormalised - // average correlations, if any locations are used - if (nused_locs > 0) { - groupCorr = avgcorr - * dnlocs - / static_cast(nused_locs); - } - } - oops::Log::trace() << "ObsErrorWithinGroupCov::recondition() end" << std::endl; -} // recondition - -// ----------------------------------------------------------------------------- - void ObsErrorWithinGroupCov::multiply(ioda::ObsVector & dy) const { // R * dy = D^{1/2} * C * D^{1/2} * dy // where D^{1/2} - diagonal matrix with stddev_ on the diagonal diff --git a/src/ufo/errors/ObsErrorWithinGroupCov.h b/src/ufo/errors/ObsErrorWithinGroupCov.h index 9314e0a2e..68089824b 100644 --- a/src/ufo/errors/ObsErrorWithinGroupCov.h +++ b/src/ufo/errors/ObsErrorWithinGroupCov.h @@ -21,7 +21,6 @@ #include "ufo/errors/ObsErrorBase.h" #include "ufo/errors/ObsErrorParametersBase.h" -#include "ufo/errors/ObsErrorReconditioner.h" namespace ioda { class ObsSpace; @@ -45,8 +44,7 @@ struct DistanceFunctionsParameterTraitsHelper { enum class CorrelationFunctions { GC99, - MARKOV, - GAUSSIAN + MARKOV }; struct CorrelationFunctionsParameterTraitsHelper { @@ -54,8 +52,7 @@ struct CorrelationFunctionsParameterTraitsHelper { static constexpr char enumTypeName[] = "CorrelationFunctions"; static constexpr util::NamedEnumerator namedValues[] = { { CorrelationFunctions::GC99, "gc99" }, - { CorrelationFunctions::MARKOV, "markov" }, - { CorrelationFunctions::GAUSSIAN, "gaussian"} + { CorrelationFunctions::MARKOV, "markov" } }; }; @@ -87,7 +84,7 @@ class ObsErrorWithinGroupCovParameters : public ObsErrorParametersBase { public: oops::Parameter correlationFunction{"correlation function", "Correlation function to use for correlation computations. " - "Currently only 'gc99', 'markov', and 'gaussian' are supported", + "Currently only 'gc99' and 'markov' are supported", CorrelationFunctions::GC99, this}; oops::RequiredParameter lscale{"correlation lengthscale", "Correlation lengthscale used with the correlation functions", this}; @@ -102,10 +99,6 @@ class ObsErrorWithinGroupCovParameters : public ObsErrorParametersBase { "The lengthscale factor is multiplied by the lengthscale to provide the limit in " "which correlations are evaluated. Beyond this distance correlation values are set to " "zero.", 1.0, this}; - oops::Parameter applyBasicReconditioning{"apply basic reconditioning", - "Apply a simple ridge regression reconditiong to the gaussian correlation profile." - "This is the same reconditioning that is applied to the markov correlation method", - false, this}; }; // ----------------------------------------------------------------------------- @@ -181,8 +174,6 @@ class ObsErrorWithinGroupCov : public ObsErrorBase { void print(std::ostream &) const override; /// Multiply only by correlations (diagnostics) void multiplyCorrelations(ioda::ObsVector & y) const; - /// Recondition the R matrix - called by update - void recondition(const ioda::ObsVector & mask); /// ObsError configuration as oops::Paramters Parameters_ params_; /// ObsSpace reference @@ -194,8 +185,6 @@ class ObsErrorWithinGroupCov : public ObsErrorBase { /// Each element holds a lower-triangle of the correlation matrix for all locations /// within one group std::vector correlations_; - /// Create reconditioner - std::unique_ptr reconditioner_; }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/filters/AcceptList.cc b/src/ufo/filters/AcceptList.cc index abda07112..f2eab5885 100644 --- a/src/ufo/filters/AcceptList.cc +++ b/src/ufo/filters/AcceptList.cc @@ -21,8 +21,8 @@ namespace ufo { // ----------------------------------------------------------------------------- AcceptList::AcceptList(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "AcceptList constructor" << std::endl; diff --git a/src/ufo/filters/AcceptList.h b/src/ufo/filters/AcceptList.h index 5a71b0582..69b922a06 100644 --- a/src/ufo/filters/AcceptList.h +++ b/src/ufo/filters/AcceptList.h @@ -54,8 +54,8 @@ class AcceptList : public FilterBase, static const std::string classname() {return "ufo::AcceptList";} AcceptList(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); private: void print(std::ostream &) const override; diff --git a/src/ufo/filters/AverageObsToGeoValLevels.cc b/src/ufo/filters/AverageObsToGeoValLevels.cc index 07416d1d4..e11923014 100644 --- a/src/ufo/filters/AverageObsToGeoValLevels.cc +++ b/src/ufo/filters/AverageObsToGeoValLevels.cc @@ -30,8 +30,8 @@ namespace ufo { AverageObsToGeoValLevels::AverageObsToGeoValLevels( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "AverageObsToGeoValLevels constructor" << std::endl; @@ -63,7 +63,7 @@ void AverageObsToGeoValLevels::applyFilter(const std::vector & apply, const Variables & filtervars, std::vector> & flagged) const { oops::Log::trace() << "AverageObsToGeoValLevels applyFilter start" << std::endl; - oops::Log::debug() << "AverageObsToGeoValLevels obserr: " << obserr_ << std::endl; + oops::Log::debug() << "AverageObsToGeoValLevels obserr: " << *obserr_ << std::endl; const float missing = util::missingValue(); ioda::ObsDataVector obs(obsdb_, filtervars.toOopsObsVariables(), "ObsValue"); diff --git a/src/ufo/filters/AverageObsToGeoValLevels.h b/src/ufo/filters/AverageObsToGeoValLevels.h index 43c338678..e4aac8b08 100644 --- a/src/ufo/filters/AverageObsToGeoValLevels.h +++ b/src/ufo/filters/AverageObsToGeoValLevels.h @@ -81,8 +81,8 @@ class AverageObsToGeoValLevels : public FilterBase, static const std::string classname() {return "ufo::AverageObsToGeoValLevels";} AverageObsToGeoValLevels(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~AverageObsToGeoValLevels(); private: diff --git a/src/ufo/filters/BackgroundCheck.cc b/src/ufo/filters/BackgroundCheck.cc index 431103087..901235f5a 100644 --- a/src/ufo/filters/BackgroundCheck.cc +++ b/src/ufo/filters/BackgroundCheck.cc @@ -29,8 +29,8 @@ namespace ufo { // ----------------------------------------------------------------------------- BackgroundCheck::BackgroundCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr, VariableNameMap(parameters.AliasFile.value())), parameters_(parameters) { @@ -94,7 +94,7 @@ void BackgroundCheck::applyFilter(const std::vector & apply, oops::Log::trace() << "BackgroundCheck applyFilter start" << std::endl; const oops::ObsVariables observed = obsdb_.obsvariables(); const float missing = util::missingValue(); - oops::Log::debug() << "BackgroundCheck obserr: " << obserr_ << std::endl; + oops::Log::debug() << "BackgroundCheck obserr: " << *obserr_ << std::endl; ioda::ObsDataVector obs(obsdb_, filtervars.toOopsObsVariables(), "ObsValue"); std::string test_hofx = parameters_.test_hofx.value(); @@ -115,10 +115,10 @@ void BackgroundCheck::applyFilter(const std::vector & apply, for (size_t jobs = 0; jobs < obsdb_.nlocs(); ++jobs) { // style note: use missingValue because obserr_ is declared in another file // and its underlying type could in principle change without it being obvious here. - const bool obserrNotMissing = (obserr_[iv][jobs] + const bool obserrNotMissing = ((*obserr_)[iv][jobs] != util::missingValue>()); - if (apply[jobs] && flags_[iv][jobs] == QCflags::pass && obserrNotMissing) { + decltype((*obserr_)[iv][jobs])>>()); + if (apply[jobs] && (*flags_)[iv][jobs] == QCflags::pass && obserrNotMissing) { ASSERT(obs[jv][jobs] != missing); ASSERT(hofx[jobs] != missing); @@ -176,16 +176,16 @@ void BackgroundCheck::applyFilter(const std::vector & apply, for (size_t jobs = 0; jobs < obsdb_.nlocs(); ++jobs) { // style note: use missingValue because obserr_ is declared in another file // and its underlying type could in principle change without it being obvious here. - const bool obserrNotMissing = (obserr_[iv][jobs] + const bool obserrNotMissing = ((*obserr_)[iv][jobs] != util::missingValue>()); - if (apply[jobs] && flags_[iv][jobs] == QCflags::pass && obserrNotMissing) { + decltype((*obserr_)[iv][jobs])>>()); + if (apply[jobs] && (*flags_)[iv][jobs] == QCflags::pass && obserrNotMissing) { ASSERT(obs[jv][jobs] != missing); ASSERT(hofx[jobs] != missing); ASSERT(bias[jobs] != missing); const std::vector &errorMultiplier = - (thresholdWrtBGerror || thresholdWrtEnsembleSpread) ? hofxerr : obserr_[iv]; + (thresholdWrtBGerror || thresholdWrtEnsembleSpread) ? hofxerr : (*obserr_)[iv]; // Threshold for current observation float zz = (thr[jobs] == std::numeric_limits::max()) ? abs_thr[jobs] : std::min(abs_thr[jobs], thr[jobs] * errorMultiplier[jobs]); diff --git a/src/ufo/filters/BackgroundCheck.h b/src/ufo/filters/BackgroundCheck.h index e4302409f..cd70c7636 100644 --- a/src/ufo/filters/BackgroundCheck.h +++ b/src/ufo/filters/BackgroundCheck.h @@ -103,8 +103,8 @@ class BackgroundCheck : public FilterBase, static const std::string classname() {return "ufo::BackgroundCheck";} BackgroundCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~BackgroundCheck(); private: diff --git a/src/ufo/filters/BayesianBackgroundCheck.cc b/src/ufo/filters/BayesianBackgroundCheck.cc index c4aacaf8c..c645c54b1 100644 --- a/src/ufo/filters/BayesianBackgroundCheck.cc +++ b/src/ufo/filters/BayesianBackgroundCheck.cc @@ -35,8 +35,8 @@ namespace ufo { BayesianBackgroundCheck::BayesianBackgroundCheck( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr, VariableNameMap(parameters.AliasFile.value())), parameters_(parameters) @@ -100,7 +100,7 @@ void BayesianBackgroundCheck::applyFilter(const std::vector & apply, oops::Log::trace() << "BayesianBackgroundCheck applyFilter start" << std::endl; const oops::ObsVariables observed = obsdb_.obsvariables(); - oops::Log::debug() << "BayesianBackgroundCheck obserr: " << obserr_ << std::endl; + oops::Log::debug() << "BayesianBackgroundCheck obserr: " << *obserr_ << std::endl; ioda::ObsDataVector obs(obsdb_, filtervars.toOopsObsVariables(), "ObsValue"); @@ -280,7 +280,7 @@ void BayesianBackgroundCheck::applyFilter(const std::vector & apply, // create reduced vectors, copied from full ones, fulfilling applycondition: std::vector firstComponentObVal_reduced = reduceVector(firstComponentObVal, j_reduced); - std::vector ObsErr_reduced = reduceVector(obserr_[varname1], j_reduced); + std::vector ObsErr_reduced = reduceVector((*obserr_)[varname1], j_reduced); std::vector hofx1_reduced = reduceVector(hofx1, j_reduced); std::vector hofxerr_reduced = reduceVector(hofxerr, j_reduced); std::vector PdBad_reduced = reduceVector(PdBad, j_reduced); diff --git a/src/ufo/filters/BayesianBackgroundCheck.h b/src/ufo/filters/BayesianBackgroundCheck.h index 2b203fae2..60c75c11f 100644 --- a/src/ufo/filters/BayesianBackgroundCheck.h +++ b/src/ufo/filters/BayesianBackgroundCheck.h @@ -132,8 +132,8 @@ class BayesianBackgroundCheck : public FilterBase, static const std::string classname() {return "ufo::BayesianBackgroundCheck";} BayesianBackgroundCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); ~BayesianBackgroundCheck(); private: diff --git a/src/ufo/filters/BayesianBackgroundQCFlags.cc b/src/ufo/filters/BayesianBackgroundQCFlags.cc index 989100435..66fbdf434 100644 --- a/src/ufo/filters/BayesianBackgroundQCFlags.cc +++ b/src/ufo/filters/BayesianBackgroundQCFlags.cc @@ -23,8 +23,8 @@ namespace ufo { BayesianBackgroundQCFlags::BayesianBackgroundQCFlags (ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "BayesianBackgroundQCFlags constructor" << std::endl; diff --git a/src/ufo/filters/BayesianBackgroundQCFlags.h b/src/ufo/filters/BayesianBackgroundQCFlags.h index 4df7e5bec..d5978920c 100644 --- a/src/ufo/filters/BayesianBackgroundQCFlags.h +++ b/src/ufo/filters/BayesianBackgroundQCFlags.h @@ -63,8 +63,8 @@ class BayesianBackgroundQCFlags : public FilterBase, static const std::string classname() {return "ufo::BayesianBackgroundQCFlags";} BayesianBackgroundQCFlags(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~BayesianBackgroundQCFlags(); private: diff --git a/src/ufo/filters/BlackList.cc b/src/ufo/filters/BlackList.cc index 6de08fd80..0105cadfc 100644 --- a/src/ufo/filters/BlackList.cc +++ b/src/ufo/filters/BlackList.cc @@ -21,8 +21,8 @@ namespace ufo { // ----------------------------------------------------------------------------- BlackList::BlackList(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "BlackList constructor" << std::endl; diff --git a/src/ufo/filters/BlackList.h b/src/ufo/filters/BlackList.h index 2275333f9..35ee576da 100644 --- a/src/ufo/filters/BlackList.h +++ b/src/ufo/filters/BlackList.h @@ -46,8 +46,8 @@ class BlackList : public FilterBase, static const std::string classname() {return "ufo::BlackList";} BlackList(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~BlackList(); private: diff --git a/src/ufo/filters/CMakeLists.txt b/src/ufo/filters/CMakeLists.txt index 45947e7a0..993bcb480 100644 --- a/src/ufo/filters/CMakeLists.txt +++ b/src/ufo/filters/CMakeLists.txt @@ -259,8 +259,6 @@ set ( filters_files obsfunctions/CloudDetectMinResidualIR.h obsfunctions/CloudFirstGuessMinimumResidual.cc obsfunctions/CloudFirstGuessMinimumResidual.h - obsfunctions/CircularDifference.cc - obsfunctions/CircularDifference.h obsfunctions/MetOfficeAllSkyErrorModel.cc obsfunctions/MetOfficeAllSkyErrorModel.h obsfunctions/ModelHeightAdjustedWindVectorComponent.cc @@ -329,8 +327,6 @@ set ( filters_files obsfunctions/SatwindIndivErrors.h obsfunctions/SatWindsLNVDCheck.cc obsfunctions/SatWindsLNVDCheck.h - obsfunctions/TimeBinner.cc - obsfunctions/TimeBinner.h obsfunctions/WindsSPDBCheck.cc obsfunctions/WindsSPDBCheck.h obsfunctions/SatWindsErrnormCheck.cc diff --git a/src/ufo/filters/ConventionalProfileProcessing.cc b/src/ufo/filters/ConventionalProfileProcessing.cc index 707b334f4..1e36e2fb6 100644 --- a/src/ufo/filters/ConventionalProfileProcessing.cc +++ b/src/ufo/filters/ConventionalProfileProcessing.cc @@ -31,8 +31,8 @@ namespace ufo { ConventionalProfileProcessing::ConventionalProfileProcessing (ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), options_(parameters) { oops::Log::trace() << "ConventionalProfileProcessing constructor" << std::endl; @@ -207,7 +207,7 @@ namespace ufo { // Handles individual profile data ProfileDataHandler profileDataHandler(data_, - flags_, + *flags_, options_.DHParameters, apply, filtervars, diff --git a/src/ufo/filters/ConventionalProfileProcessing.h b/src/ufo/filters/ConventionalProfileProcessing.h index 083fa9aea..15f93981c 100644 --- a/src/ufo/filters/ConventionalProfileProcessing.h +++ b/src/ufo/filters/ConventionalProfileProcessing.h @@ -86,8 +86,8 @@ namespace ufo { static const std::string classname() {return "ufo::ConventionalProfileProcessing";} ConventionalProfileProcessing(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ConventionalProfileProcessing(); /// Return the number of mismatches between values produced by the checking routines diff --git a/src/ufo/filters/CopyFlagsFromExtendedToOriginalSpace.cc b/src/ufo/filters/CopyFlagsFromExtendedToOriginalSpace.cc index 9bf95d8ac..3dcb70260 100644 --- a/src/ufo/filters/CopyFlagsFromExtendedToOriginalSpace.cc +++ b/src/ufo/filters/CopyFlagsFromExtendedToOriginalSpace.cc @@ -19,8 +19,8 @@ namespace ufo { CopyFlagsFromExtendedToOriginalSpace::CopyFlagsFromExtendedToOriginalSpace (ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr> flags, + std::shared_ptr> obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "CopyFlagsFromExtendedToOriginalSpace constructor start" << std::endl; @@ -55,7 +55,7 @@ void CopyFlagsFromExtendedToOriginalSpace::applyFilter(const std::vector & const Variables & filtervars, std::vector> & flagged) const { oops::Log::trace() << "CopyFlagsFromExtendedToOriginalSpace applyFilter start" << std::endl; - oops::Log::debug() << "CopyFlagsFromExtendedToOriginalSpace obserr: " << obserr_ << std::endl; + oops::Log::debug() << "CopyFlagsFromExtendedToOriginalSpace obserr: " << *obserr_ << std::endl; // Number of locations. const size_t nlocs = obsdb_.nlocs(); diff --git a/src/ufo/filters/CopyFlagsFromExtendedToOriginalSpace.h b/src/ufo/filters/CopyFlagsFromExtendedToOriginalSpace.h index 9125373dc..37fcb43ab 100644 --- a/src/ufo/filters/CopyFlagsFromExtendedToOriginalSpace.h +++ b/src/ufo/filters/CopyFlagsFromExtendedToOriginalSpace.h @@ -71,8 +71,8 @@ class CopyFlagsFromExtendedToOriginalSpace : public FilterBase, static const std::string classname() {return "ufo::CopyFlagsFromExtendedToOriginalSpace";} CopyFlagsFromExtendedToOriginalSpace(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr> flags, + std::shared_ptr> obserr); ~CopyFlagsFromExtendedToOriginalSpace() override; private: diff --git a/src/ufo/filters/CreateDiagnosticFlags.cc b/src/ufo/filters/CreateDiagnosticFlags.cc index 6217e7db9..1adbf5006 100644 --- a/src/ufo/filters/CreateDiagnosticFlags.cc +++ b/src/ufo/filters/CreateDiagnosticFlags.cc @@ -22,9 +22,9 @@ namespace ufo { // ----------------------------------------------------------------------------- CreateDiagnosticFlags::CreateDiagnosticFlags(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) - : ObsProcessorBase(obsdb, parameters.deferToPost, flags, obserr), + std::shared_ptr> qcflags, + std::shared_ptr> obserr) + : ObsProcessorBase(obsdb, parameters.deferToPost, std::move(qcflags), std::move(obserr)), parameters_(parameters) { oops::Log::trace() << "CreateDiagnosticFlags constructor start" << std::endl; diff --git a/src/ufo/filters/CreateDiagnosticFlags.h b/src/ufo/filters/CreateDiagnosticFlags.h index 2e43a8aca..4cf5d339b 100644 --- a/src/ufo/filters/CreateDiagnosticFlags.h +++ b/src/ufo/filters/CreateDiagnosticFlags.h @@ -98,8 +98,8 @@ class CreateDiagnosticFlags : public ObsProcessorBase, static const std::string classname() {return "ufo::CreateDiagnosticFlags";} CreateDiagnosticFlags(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr> qcflags, + std::shared_ptr> obserr); ~CreateDiagnosticFlags() override; private: diff --git a/src/ufo/filters/DifferenceCheck.cc b/src/ufo/filters/DifferenceCheck.cc index 80ef13e61..c4d3b9cc7 100644 --- a/src/ufo/filters/DifferenceCheck.cc +++ b/src/ufo/filters/DifferenceCheck.cc @@ -60,8 +60,8 @@ void flagWhereDiffOut(const std::vector &refValues, std::vector &t // ----------------------------------------------------------------------------- DifferenceCheck::DifferenceCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { diff --git a/src/ufo/filters/DifferenceCheck.h b/src/ufo/filters/DifferenceCheck.h index 8f82b7560..05ede00c4 100644 --- a/src/ufo/filters/DifferenceCheck.h +++ b/src/ufo/filters/DifferenceCheck.h @@ -69,8 +69,8 @@ class DifferenceCheck : public FilterBase, static const std::string classname() {return "ufo::DifferenceCheck";} DifferenceCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~DifferenceCheck(); private: diff --git a/src/ufo/filters/EnsembleStatistics.cc b/src/ufo/filters/EnsembleStatistics.cc index a7699204f..0d3ecc2e3 100644 --- a/src/ufo/filters/EnsembleStatistics.cc +++ b/src/ufo/filters/EnsembleStatistics.cc @@ -55,9 +55,9 @@ eckit::mpi::Comm & getOrCreateEnsembleComm(size_t geoCommSize, size_t timeCommSi // ----------------------------------------------------------------------------- EnsembleStatistics::EnsembleStatistics(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) -: ObsProcessorBase(obsdb, true /*deferToPost?*/, flags, obserr), + std::shared_ptr > flags, + std::shared_ptr > obserr) +: ObsProcessorBase(obsdb, true /*deferToPost?*/, std::move(flags), std::move(obserr)), parameters_(parameters) { oops::Log::trace() << "EnsembleStatistics constructor start" << std::endl; diff --git a/src/ufo/filters/EnsembleStatistics.h b/src/ufo/filters/EnsembleStatistics.h index 7f5a4ae61..33f18caef 100644 --- a/src/ufo/filters/EnsembleStatistics.h +++ b/src/ufo/filters/EnsembleStatistics.h @@ -89,8 +89,8 @@ class EnsembleStatistics : public ObsProcessorBase, static const std::string classname() {return "ufo::EnsembleStatistics";} EnsembleStatistics(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~EnsembleStatistics() override; private: diff --git a/src/ufo/filters/FilterBase.cc b/src/ufo/filters/FilterBase.cc index 9124824d5..88237d39b 100644 --- a/src/ufo/filters/FilterBase.cc +++ b/src/ufo/filters/FilterBase.cc @@ -30,10 +30,10 @@ namespace ufo { FilterBase::FilterBase(ioda::ObsSpace & os, const FilterParametersBaseWithAbstractActions & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr, + std::shared_ptr > flags, + std::shared_ptr > obserr, const VariableNameMap & nameMap) - : ObsProcessorBase(os, parameters.deferToPost, flags, obserr), + : ObsProcessorBase(os, parameters.deferToPost, std::move(flags), std::move(obserr)), filtervars_(), nameMap_(nameMap), whereParameters_(parameters.where), @@ -75,13 +75,13 @@ FilterBase::FilterBase(ioda::ObsSpace & os, // ----------------------------------------------------------------------------- FilterBase::FilterBase(ioda::ObsSpace & os, const eckit::Configuration & config, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr, + std::shared_ptr > flags, + std::shared_ptr > obserr, const VariableNameMap & nameMap) : FilterBase(os, oops::validateAndDeserialize(config), - flags, - obserr, + std::move(flags), + std::move(obserr), nameMap) {} @@ -127,7 +127,7 @@ void FilterBase::doFilter() { // Take actions for (const std::unique_ptr &actionParameters : actionsParameters_) { FilterAction action(*actionParameters); - action.apply(vars, flagged, data_, this->qcFlag(), flags_, obserr_); + action.apply(vars, flagged, data_, this->qcFlag(), *flags_, *obserr_); } // Done diff --git a/src/ufo/filters/FilterBase.h b/src/ufo/filters/FilterBase.h index 5a1d6913f..cd41fef94 100644 --- a/src/ufo/filters/FilterBase.h +++ b/src/ufo/filters/FilterBase.h @@ -42,8 +42,8 @@ namespace ufo { /// In the former case, `Filter` should provide a constructor with the following signature: /// /// Filter(ioda::ObsSpace &os, const eckit::Configuration &conf, -/// ioda::ObsDataVector &flags, -/// ioda::ObsDataVector &obserr); +/// std::shared_ptr > qcflags, +/// std::shared_ptr > obserrors); /// /// In the latter case, a subclass of FilterParametersBaseWithAbstractAction holding the settings /// of the filter in question should be defined. (Usually it can in fact be a subclass of @@ -53,18 +53,18 @@ namespace ufo { /// of that subclass and provide a constructor with the following signature: /// /// Filter(ioda::ObsSpace &os, const Parameters_ ¶ms, -/// ioda::ObsDataVector &flags, -/// ioda::ObsDataVector &obserr); +/// std::shared_ptr > qcflags, +/// std::shared_ptr > obserrors); /// class FilterBase : public ObsProcessorBase { public: FilterBase(ioda::ObsSpace &, const FilterParametersBaseWithAbstractActions ¶meters, - ioda::ObsDataVector &, - ioda::ObsDataVector &, + std::shared_ptr >, + std::shared_ptr >, const VariableNameMap & nameMap = VariableNameMap(boost::none)); FilterBase(ioda::ObsSpace &, const eckit::Configuration &, - ioda::ObsDataVector &, - ioda::ObsDataVector &, + std::shared_ptr >, + std::shared_ptr >, const VariableNameMap & nameMap = VariableNameMap(boost::none)); ~FilterBase(); diff --git a/src/ufo/filters/FinalCheck.cc b/src/ufo/filters/FinalCheck.cc index 7be7246eb..16c9bbf73 100644 --- a/src/ufo/filters/FinalCheck.cc +++ b/src/ufo/filters/FinalCheck.cc @@ -22,10 +22,9 @@ namespace ufo { // ----------------------------------------------------------------------------- FinalCheck::FinalCheck(ioda::ObsSpace & obsdb, const Parameters_ &, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) - - : ObsProcessorBase(obsdb, true /*deferToPost?*/, flags, obserr) + std::shared_ptr> qcflags, + std::shared_ptr> obserr) + : ObsProcessorBase(obsdb, true /*deferToPost?*/, std::move(qcflags), std::move(obserr)) { oops::Log::trace() << "FinalCheck constructor" << std::endl; } @@ -54,8 +53,8 @@ void FinalCheck::doFilter() { const float missing = util::missingValue(); for (size_t jv = 0; jv < obsdb_.obsvariables().size(); ++jv) { for (size_t jobs = 0; jobs < obsdb_.nlocs(); ++jobs) { - if (flags_[jv][jobs] == QCflags::pass && obserr_[jv][jobs] == missing) { - flags_[jv][jobs] = QCflags::missing; + if ((*flags_)[jv][jobs] == QCflags::pass && (*obserr_)[jv][jobs] == missing) { + (*flags_)[jv][jobs] = QCflags::missing; } } } @@ -65,7 +64,7 @@ void FinalCheck::doFilter() { for (size_t jv = 0; jv < obsdb_.obsvariables().size(); ++jv) { if (!obsdb_.assimvariables().has(obsdb_.obsvariables()[jv])) { for (size_t jobs = 0; jobs < obsdb_.nlocs(); ++jobs) { - flags_[jv][jobs] = QCflags::processed; + (*flags_)[jv][jobs] = QCflags::processed; } } } diff --git a/src/ufo/filters/FinalCheck.h b/src/ufo/filters/FinalCheck.h index 803a17967..09b6dbecd 100644 --- a/src/ufo/filters/FinalCheck.h +++ b/src/ufo/filters/FinalCheck.h @@ -42,8 +42,8 @@ class FinalCheck : public ObsProcessorBase, static const std::string classname() {return "ufo::FinalCheck";} FinalCheck(ioda::ObsSpace & obsdb, const Parameters_ & params, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr> qcflags, + std::shared_ptr> obserr); ~FinalCheck() override; void doFilter() override; diff --git a/src/ufo/filters/Gaussian_Thinning.cc b/src/ufo/filters/Gaussian_Thinning.cc index 96d824974..244148b7c 100644 --- a/src/ufo/filters/Gaussian_Thinning.cc +++ b/src/ufo/filters/Gaussian_Thinning.cc @@ -43,8 +43,8 @@ namespace ufo { Gaussian_Thinning::Gaussian_Thinning(ioda::ObsSpace & obsdb, const GaussianThinningParameters & params, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, params, flags, obserr), options_(params) { oops::Log::trace() << "Gaussian_Thinning constructor" << std::endl; oops::Log::debug() << "Gaussian_Thinning: config = " << options_ << std::endl; @@ -66,7 +66,7 @@ void Gaussian_Thinning::applyFilter(const std::vector & apply, // returns what it has been passed without modification. const RecordHandler recordHandler(obsdb_, filtervars, - flags_, + *flags_, retainOnlyIfAllFilterVariablesAreValid); // If records are treated as single obs and a category variable is also used, @@ -87,7 +87,7 @@ void Gaussian_Thinning::applyFilter(const std::vector & apply, obsAccessor.getValidObservationIds(options_.recordsAreSingleObs ? recordHandler.changeApplyIfRecordsAreSingleObs(apply) : apply, - flags_, + *flags_, filtervars, !retainOnlyIfAllFilterVariablesAreValid); } @@ -231,7 +231,7 @@ void Gaussian_Thinning::applyFilter(const std::vector & apply, // Optionally reject all filter variables if any has failed QC and ob is invalid for thinning if (retainOnlyIfAllFilterVariablesAreValid) - obsAccessor.flagObservationsForAnyFilterVariableFailingQC(apply, flags_, filtervars, flagged); + obsAccessor.flagObservationsForAnyFilterVariableFailingQC(apply, *flags_, filtervars, flagged); oops::Log::trace() << "Gaussian_Thinning applyFilter complete" << std::endl; } diff --git a/src/ufo/filters/Gaussian_Thinning.h b/src/ufo/filters/Gaussian_Thinning.h index c1abb66ec..f58d54596 100644 --- a/src/ufo/filters/Gaussian_Thinning.h +++ b/src/ufo/filters/Gaussian_Thinning.h @@ -62,8 +62,8 @@ class Gaussian_Thinning : public FilterBase, static const std::string classname() {return "ufo::Gaussian_Thinning";} Gaussian_Thinning(ioda::ObsSpace &obsdb, const GaussianThinningParameters ¶ms, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); private: void print(std::ostream &) const override; diff --git a/src/ufo/filters/GeoVaLsWriter.cc b/src/ufo/filters/GeoVaLsWriter.cc index 4a37a8271..d6c05ec05 100644 --- a/src/ufo/filters/GeoVaLsWriter.cc +++ b/src/ufo/filters/GeoVaLsWriter.cc @@ -18,7 +18,7 @@ namespace ufo { // ----------------------------------------------------------------------------- GeoVaLsWriter::GeoVaLsWriter(const ioda::ObsSpace &, const Parameters_ & params, - ioda::ObsDataVector &, ioda::ObsDataVector &) + ObsDataPtr_, ObsDataPtr_) : config_(params.config), novars_() { oops::Log::trace() << "GeoVaLsWriter constructor" << std::endl; } diff --git a/src/ufo/filters/GeoVaLsWriter.h b/src/ufo/filters/GeoVaLsWriter.h index 7adeedf76..9f3f06fd7 100644 --- a/src/ufo/filters/GeoVaLsWriter.h +++ b/src/ufo/filters/GeoVaLsWriter.h @@ -29,11 +29,13 @@ class ObsDiagnostics; // ----------------------------------------------------------------------------- class GeoVaLsWriter : public ObsFilterBase { + template using ObsDataPtr_ = std::shared_ptr >; + public: typedef GenericFilterParameters Parameters_; GeoVaLsWriter(const ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, ioda::ObsDataVector &); + ObsDataPtr_, ObsDataPtr_); ~GeoVaLsWriter() = default; void preProcess() override {} diff --git a/src/ufo/filters/HistoryCheck.cc b/src/ufo/filters/HistoryCheck.cc index 4f7928f00..7590dc13b 100644 --- a/src/ufo/filters/HistoryCheck.cc +++ b/src/ufo/filters/HistoryCheck.cc @@ -38,8 +38,8 @@ namespace ufo { HistoryCheck::HistoryCheck(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), options_(parameters) { oops::Log::debug() << "HistoryCheck: config = " << options_ << "\n"; @@ -47,8 +47,8 @@ HistoryCheck::HistoryCheck(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, HistoryCheck::HistoryCheck(ioda::ObsSpace &obsdb, const ufo::HistoryCheck::Parameters_ ¶meters, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr, + std::shared_ptr > flags, + std::shared_ptr > obserr, const eckit::LocalConfiguration &conf) : HistoryCheck(obsdb, parameters, flags, obserr) { @@ -93,8 +93,10 @@ void HistoryCheck::applyFilter(const std::vector & apply, widerObsSpace.put_db("ObsValue", "airTemperature", airTemperatures); } } // end of manual data entry section used for unit testing - ioda::ObsDataVector obserrWide(widerObsSpace, widerObsSpace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflagsWide(widerObsSpace, widerObsSpace.obsvariables()); + std::shared_ptr> obserrWide( + new ioda::ObsDataVector(widerObsSpace, widerObsSpace.obsvariables(), "ObsError")); + std::shared_ptr> qcflagsWide( + new ioda::ObsDataVector(widerObsSpace, widerObsSpace.obsvariables())); const oops::RequiredParameter &subtype = options_.surfaceObservationSubtype; const boost::optional &trackOptions = @@ -196,7 +198,7 @@ void HistoryCheck::applyFilter(const std::vector & apply, // counter" until the full identifier to-be-added is unique within the set. std::set wideFlaggedLocationIds; // qc flags are the same across all variables for these filters - const std::vector &wideFlags = qcflagsWide[0]; + const std::vector &wideFlags = (*qcflagsWide)[0]; for (size_t i = 0; i < wideFlags.size(); i++) { if (wideFlags[i] == QCflags::track) { obsIdentifierData obsLabel = { diff --git a/src/ufo/filters/HistoryCheck.h b/src/ufo/filters/HistoryCheck.h index 5f8742ad6..837c727c5 100644 --- a/src/ufo/filters/HistoryCheck.h +++ b/src/ufo/filters/HistoryCheck.h @@ -41,13 +41,13 @@ class HistoryCheck: public FilterBase, /// 2. Apply track check and stuck value check over wider window /// 3. Apply flags over wider window to observations in main window. HistoryCheck(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); // This constructor is needed for unit testing HistoryCheck(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr, + std::shared_ptr > flags, + std::shared_ptr > obserr, const eckit::LocalConfiguration &conf); private: diff --git a/src/ufo/filters/ImpactHeightCheck.cc b/src/ufo/filters/ImpactHeightCheck.cc index ee62807ea..1399dde68 100644 --- a/src/ufo/filters/ImpactHeightCheck.cc +++ b/src/ufo/filters/ImpactHeightCheck.cc @@ -30,8 +30,8 @@ namespace ufo { ImpactHeightCheck::ImpactHeightCheck( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "ImpactHeightCheck constructor" << std::endl; @@ -250,7 +250,7 @@ void ImpactHeightCheck::applyFilter(const std::vector & apply, // Loop over all observations in the profile for (size_t jobs : obs_numbers) { // Check that this observation should be considered in this routine - if (apply[jobs] && flags_[iVar][jobs] == QCflags::pass) { + if (apply[jobs] && (*flags_)[iVar][jobs] == QCflags::pass) { // Reject observation if it is below the minimum (either surface or sharp gradient) const float obsImpactHeight = impactParameter[iVar][jobs] - radiusCurvature[jobs]; if (parameters_.verboseOutput.value()) diff --git a/src/ufo/filters/ImpactHeightCheck.h b/src/ufo/filters/ImpactHeightCheck.h index 04499ed56..c226fc1ef 100644 --- a/src/ufo/filters/ImpactHeightCheck.h +++ b/src/ufo/filters/ImpactHeightCheck.h @@ -71,8 +71,8 @@ class ImpactHeightCheck : public FilterBase, static const std::string classname() {return "ufo::ImpactHeightCheck";} ImpactHeightCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ImpactHeightCheck(); private: diff --git a/src/ufo/filters/MWCLWCheck.cc b/src/ufo/filters/MWCLWCheck.cc index f5d8a8966..3270e2814 100644 --- a/src/ufo/filters/MWCLWCheck.cc +++ b/src/ufo/filters/MWCLWCheck.cc @@ -26,8 +26,8 @@ namespace ufo { // ----------------------------------------------------------------------------- MWCLWCheck::MWCLWCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "MWCLWCheck constructor" << std::endl; oops::Log::debug() << "MWCLWCheck: config = " << parameters_ << std::endl; diff --git a/src/ufo/filters/MWCLWCheck.h b/src/ufo/filters/MWCLWCheck.h index ff1eb875d..2755c8436 100644 --- a/src/ufo/filters/MWCLWCheck.h +++ b/src/ufo/filters/MWCLWCheck.h @@ -60,8 +60,8 @@ class MWCLWCheck : public FilterBase, static const std::string classname() {return "ufo::MWCLWCheck";} MWCLWCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~MWCLWCheck(); private: diff --git a/src/ufo/filters/MetOfficeBuddyCheck.cc b/src/ufo/filters/MetOfficeBuddyCheck.cc index edfb13c25..69a4a0226 100644 --- a/src/ufo/filters/MetOfficeBuddyCheck.cc +++ b/src/ufo/filters/MetOfficeBuddyCheck.cc @@ -283,9 +283,9 @@ struct MetOfficeBuddyCheck::MetaData { }; MetOfficeBuddyCheck::MetOfficeBuddyCheck(ioda::ObsSpace& obsdb, const Parameters_& parameters, - ioda::ObsDataVector& flags, - ioda::ObsDataVector& obserr) - : FilterBase(obsdb, parameters, flags, obserr, + std::shared_ptr > flags, + std::shared_ptr > obserr) + : FilterBase(obsdb, parameters, std::move(flags), std::move(obserr), VariableNameMap(parameters.AliasFile.value())), options_(parameters) { oops::Log::trace() << "MetOfficeBuddyCheck constructor" << std::endl; @@ -999,7 +999,7 @@ std::vector MetOfficeBuddyCheck::getValidObservationIds( const std::vector & apply, const boost::optional & profileIndex) const { std::vector isValid = apply; - unselectRejectedLocations(isValid, filtervars_, flags_, + unselectRejectedLocations(isValid, filtervars_, *flags_, UnselectLocationIf::ALL_FILTER_VARIABLES_REJECTED); std::vector isValidAsInt(apply.begin(), apply.end()); diff --git a/src/ufo/filters/MetOfficeBuddyCheck.h b/src/ufo/filters/MetOfficeBuddyCheck.h index d39894b2d..2c711e31d 100644 --- a/src/ufo/filters/MetOfficeBuddyCheck.h +++ b/src/ufo/filters/MetOfficeBuddyCheck.h @@ -85,8 +85,8 @@ class MetOfficeBuddyCheck : public FilterBase, static const std::string classname() {return "ufo::MetOfficeBuddyCheck";} MetOfficeBuddyCheck(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); private: struct MetaData; diff --git a/src/ufo/filters/MetOfficeDuplicateCheck.cc b/src/ufo/filters/MetOfficeDuplicateCheck.cc index 607fa74c5..5981b4ba5 100644 --- a/src/ufo/filters/MetOfficeDuplicateCheck.cc +++ b/src/ufo/filters/MetOfficeDuplicateCheck.cc @@ -18,8 +18,8 @@ namespace ufo { MetOfficeDuplicateCheck::MetOfficeDuplicateCheck (ioda::ObsSpace & obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), options_(parameters) { oops::Log::trace() << "MetOfficeDuplicateCheck constructor" << std::endl; oops::Log::debug() << "MetOfficeDuplicateCheck: config = " << options_ << std::endl; diff --git a/src/ufo/filters/MetOfficeDuplicateCheck.h b/src/ufo/filters/MetOfficeDuplicateCheck.h index eacc64111..2336956c9 100644 --- a/src/ufo/filters/MetOfficeDuplicateCheck.h +++ b/src/ufo/filters/MetOfficeDuplicateCheck.h @@ -107,8 +107,8 @@ class MetOfficeDuplicateCheck : public FilterBase, static const std::string classname() {return "ufo::MetOfficeDuplicateCheck";} MetOfficeDuplicateCheck(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); ~MetOfficeDuplicateCheck() override; diff --git a/src/ufo/filters/MetOfficePressureConsistencyCheck.cc b/src/ufo/filters/MetOfficePressureConsistencyCheck.cc index 8900e6306..7695f1fe7 100644 --- a/src/ufo/filters/MetOfficePressureConsistencyCheck.cc +++ b/src/ufo/filters/MetOfficePressureConsistencyCheck.cc @@ -18,8 +18,8 @@ namespace ufo { MetOfficePressureConsistencyCheck::MetOfficePressureConsistencyCheck (ioda::ObsSpace & obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), options_(parameters) { oops::Log::trace() << "MetOfficePressureConsistencyCheck constructor" << std::endl; oops::Log::debug() << "MetOfficePressureConsistencyCheck: config = " << options_ << std::endl; @@ -37,7 +37,7 @@ void MetOfficePressureConsistencyCheck::applyFilter(const std::vector & ap const ObsAccessor obsAccessor = createObsAccessor(); const std::vector validObsIds = - obsAccessor.getValidObservationIds(apply, flags_, filtervars); + obsAccessor.getValidObservationIds(apply, *flags_, filtervars); RecursiveSplitter splitter = obsAccessor.splitObservationsIntoIndependentGroups(validObsIds); diff --git a/src/ufo/filters/MetOfficePressureConsistencyCheck.h b/src/ufo/filters/MetOfficePressureConsistencyCheck.h index 5e1e34a99..c64e5fbf6 100644 --- a/src/ufo/filters/MetOfficePressureConsistencyCheck.h +++ b/src/ufo/filters/MetOfficePressureConsistencyCheck.h @@ -56,8 +56,8 @@ class MetOfficePressureConsistencyCheck : public FilterBase, static const std::string classname() {return "ufo::MetOfficePressureConsistencyCheck";} MetOfficePressureConsistencyCheck(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); ~MetOfficePressureConsistencyCheck() override; diff --git a/src/ufo/filters/ModelBestFitPressure.cc b/src/ufo/filters/ModelBestFitPressure.cc index 6238855c3..7dc64059a 100644 --- a/src/ufo/filters/ModelBestFitPressure.cc +++ b/src/ufo/filters/ModelBestFitPressure.cc @@ -25,8 +25,8 @@ namespace ufo { // ----------------------------------------------------------------------------- ModelBestFitPressure::ModelBestFitPressure(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { diff --git a/src/ufo/filters/ModelBestFitPressure.h b/src/ufo/filters/ModelBestFitPressure.h index 77fb4635b..a15fc0d7c 100644 --- a/src/ufo/filters/ModelBestFitPressure.h +++ b/src/ufo/filters/ModelBestFitPressure.h @@ -93,8 +93,8 @@ class ModelBestFitPressure : public FilterBase, static const std::string classname() {return "ufo::ModelBestFitPressure";} ModelBestFitPressure(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ModelBestFitPressure(); private: diff --git a/src/ufo/filters/ModelObThreshold.cc b/src/ufo/filters/ModelObThreshold.cc index 1b54e73bd..f83f109bf 100644 --- a/src/ufo/filters/ModelObThreshold.cc +++ b/src/ufo/filters/ModelObThreshold.cc @@ -25,8 +25,8 @@ constexpr util::NamedEnumerator // ----------------------------------------------------------------------------- ModelObThreshold::ModelObThreshold(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "ModelObThreshold constructor" << std::endl; diff --git a/src/ufo/filters/ModelObThreshold.h b/src/ufo/filters/ModelObThreshold.h index c6e39b398..1dc5c9c55 100644 --- a/src/ufo/filters/ModelObThreshold.h +++ b/src/ufo/filters/ModelObThreshold.h @@ -91,8 +91,8 @@ class ModelObThreshold : public FilterBase, static const std::string classname() {return "ufo::ModelObThreshold";} ModelObThreshold(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ModelObThreshold(); private: diff --git a/src/ufo/filters/ObsAccessor.cc b/src/ufo/filters/ObsAccessor.cc index 5c5bff459..803592c72 100644 --- a/src/ufo/filters/ObsAccessor.cc +++ b/src/ufo/filters/ObsAccessor.cc @@ -416,9 +416,6 @@ RecursiveSplitter ObsAccessor::splitObservationsIntoIndependentGroups( case GroupBy::SINGLE_OBS: groupObservationsByCategoryVariable(validObsIds, splitter); break; - case GroupBy::RECORD_ID: - groupObservationsByRecordNumber(validObsIds, splitter); - break; } return splitter; } diff --git a/src/ufo/filters/ObsBoundsCheck.cc b/src/ufo/filters/ObsBoundsCheck.cc index e566bc164..c41b8fd9a 100644 --- a/src/ufo/filters/ObsBoundsCheck.cc +++ b/src/ufo/filters/ObsBoundsCheck.cc @@ -67,8 +67,8 @@ void flagWhereOutOfBounds(const std::vector & apply, // ----------------------------------------------------------------------------- ObsBoundsCheck::ObsBoundsCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "ObsBoundsCheck constructor" << std::endl; @@ -143,7 +143,7 @@ void ObsBoundsCheck::applyFilter(const std::vector & apply, std::vector testAtLocations = apply; if (onlyTestGoodFilterVarsForFlagAllFilterVars) { for (size_t iloc=0; iloc < testAtLocations.size(); iloc++) - if (flags_[filtervarslist[ifiltervar]][iloc] != QCflags::pass) { + if ((*flags_)[filtervarslist[ifiltervar]][iloc] != QCflags::pass) { testAtLocations[iloc] = false; } } diff --git a/src/ufo/filters/ObsBoundsCheck.h b/src/ufo/filters/ObsBoundsCheck.h index e6b335a69..720d1ab00 100644 --- a/src/ufo/filters/ObsBoundsCheck.h +++ b/src/ufo/filters/ObsBoundsCheck.h @@ -100,8 +100,8 @@ class ObsBoundsCheck : public FilterBase, static const std::string classname() {return "ufo::ObsBoundsCheck";} ObsBoundsCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ObsBoundsCheck(); private: diff --git a/src/ufo/filters/ObsDerivativeCheck.cc b/src/ufo/filters/ObsDerivativeCheck.cc index fd377df23..8ca306a3d 100644 --- a/src/ufo/filters/ObsDerivativeCheck.cc +++ b/src/ufo/filters/ObsDerivativeCheck.cc @@ -28,8 +28,8 @@ namespace ufo { // ----------------------------------------------------------------------------- ObsDerivativeCheck::ObsDerivativeCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { diff --git a/src/ufo/filters/ObsDerivativeCheck.h b/src/ufo/filters/ObsDerivativeCheck.h index 2cb191c33..94596ef2b 100644 --- a/src/ufo/filters/ObsDerivativeCheck.h +++ b/src/ufo/filters/ObsDerivativeCheck.h @@ -83,8 +83,8 @@ class ObsDerivativeCheck : public FilterBase, static const std::string classname() {return "ufo::ObsDerivativeCheck";} ObsDerivativeCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ObsDerivativeCheck(); private: diff --git a/src/ufo/filters/ObsDiagnosticsWriter.cc b/src/ufo/filters/ObsDiagnosticsWriter.cc index 78ea048d4..f073b22fb 100644 --- a/src/ufo/filters/ObsDiagnosticsWriter.cc +++ b/src/ufo/filters/ObsDiagnosticsWriter.cc @@ -16,8 +16,8 @@ namespace ufo { ObsDiagnosticsWriter::ObsDiagnosticsWriter( ioda::ObsSpace &, const Parameters_ & params, - ioda::ObsDataVector &, - ioda::ObsDataVector &) + std::shared_ptr >, + std::shared_ptr >) : params_(params), extradiagvars_() { oops::Log::trace() << "ObsDiagnosticsWriter constructor" << std::endl; diff --git a/src/ufo/filters/ObsDiagnosticsWriter.h b/src/ufo/filters/ObsDiagnosticsWriter.h index f73add41f..f79477045 100644 --- a/src/ufo/filters/ObsDiagnosticsWriter.h +++ b/src/ufo/filters/ObsDiagnosticsWriter.h @@ -43,8 +43,8 @@ class ObsDiagnosticsWriter : public ObsFilterBase { public: typedef ObsDiagnosticsWriterParameters Parameters_; ObsDiagnosticsWriter(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ObsDiagnosticsWriter() {} void preProcess() override {} diff --git a/src/ufo/filters/ObsDomainCheck.cc b/src/ufo/filters/ObsDomainCheck.cc index 6a9e3262b..c702d98bd 100644 --- a/src/ufo/filters/ObsDomainCheck.cc +++ b/src/ufo/filters/ObsDomainCheck.cc @@ -17,8 +17,8 @@ namespace ufo { // ----------------------------------------------------------------------------- ObsDomainCheck::ObsDomainCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "ObsDomainCheck constructor" << std::endl; diff --git a/src/ufo/filters/ObsDomainCheck.h b/src/ufo/filters/ObsDomainCheck.h index 40aad8a0e..20bea2523 100644 --- a/src/ufo/filters/ObsDomainCheck.h +++ b/src/ufo/filters/ObsDomainCheck.h @@ -48,8 +48,8 @@ class ObsDomainCheck : public FilterBase, static const std::string classname() {return "ufo::ObsDomainCheck";} ObsDomainCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ObsDomainCheck(); private: diff --git a/src/ufo/filters/ObsDomainErrCheck.cc b/src/ufo/filters/ObsDomainErrCheck.cc index feb3304dc..fcc2685a1 100644 --- a/src/ufo/filters/ObsDomainErrCheck.cc +++ b/src/ufo/filters/ObsDomainErrCheck.cc @@ -24,13 +24,14 @@ namespace ufo { // ----------------------------------------------------------------------------- ObsDomainErrCheck::ObsDomainErrCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "ObsDomainErrCheck constructor" << std::endl; oops::Log::debug() << "ObsDomainErrCheck: config = " << parameters_ << std::endl; + ASSERT(obserr); } // ----------------------------------------------------------------------------- @@ -69,13 +70,13 @@ void ObsDomainErrCheck::applyFilter(const std::vector & inside, } else { // style note: use missingValue because obserr_ is declared in another file // and its underlying type could in principle change without it being obvious here. - ASSERT(obserr_[iv][jobs] != util::missingValue>()); + ASSERT((*obserr_)[iv][jobs] != util::missingValue>()); ASSERT(obs[jv][jobs] != util::missingValue()); - float bound = 2.5 * obserr_[iv][jobs]; - float obserrinc = parameter * std::max((values[jobs]-9.0), 0.0) * obserr_[iv][jobs]; - obserrinc = std::max(obserr_[iv][jobs], bound); - obserr_[iv][jobs] = sqrt(pow(obserr_[iv][jobs], 2) + pow(obserrinc, 2)); + float bound = 2.5 * (*obserr_)[iv][jobs]; + float obserrinc = parameter * std::max((values[jobs]-9.0), 0.0) * (*obserr_)[iv][jobs]; + obserrinc = std::max((*obserr_)[iv][jobs], bound); + (*obserr_)[iv][jobs] = sqrt(pow((*obserr_)[iv][jobs], 2) + pow(obserrinc, 2)); ++count; } } diff --git a/src/ufo/filters/ObsDomainErrCheck.h b/src/ufo/filters/ObsDomainErrCheck.h index f5311202f..4fa9a4a90 100644 --- a/src/ufo/filters/ObsDomainErrCheck.h +++ b/src/ufo/filters/ObsDomainErrCheck.h @@ -45,8 +45,8 @@ class ObsDomainErrCheck : public FilterBase, static const std::string classname() {return "ufo::ObsDomainErrCheck";} ObsDomainErrCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ObsDomainErrCheck(); private: diff --git a/src/ufo/filters/ObsPolygonCheck.cc b/src/ufo/filters/ObsPolygonCheck.cc index d8c4bf4c4..ebb11845b 100644 --- a/src/ufo/filters/ObsPolygonCheck.cc +++ b/src/ufo/filters/ObsPolygonCheck.cc @@ -30,8 +30,8 @@ namespace ufo { // ----------------------------------------------------------------------------- ObsPolygonCheck::ObsPolygonCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "ObsPolygonCheck constructor" << std::endl; diff --git a/src/ufo/filters/ObsPolygonCheck.h b/src/ufo/filters/ObsPolygonCheck.h index 34ef91fd9..153b9d6f5 100644 --- a/src/ufo/filters/ObsPolygonCheck.h +++ b/src/ufo/filters/ObsPolygonCheck.h @@ -70,8 +70,8 @@ class ObsPolygonCheck : public FilterBase, static const std::string classname() {return "ufo::ObsPolygonCheck";} ObsPolygonCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ObsPolygonCheck(); private: diff --git a/src/ufo/filters/ObsProcessorBase.cc b/src/ufo/filters/ObsProcessorBase.cc index a6d77d795..61e2d9216 100644 --- a/src/ufo/filters/ObsProcessorBase.cc +++ b/src/ufo/filters/ObsProcessorBase.cc @@ -27,16 +27,18 @@ namespace ufo { // ----------------------------------------------------------------------------- ObsProcessorBase::ObsProcessorBase(ioda::ObsSpace & os, bool deferToPost, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : obsdb_(os), flags_(flags), obserr_(obserr), data_(obsdb_), prior_(false), post_(false), deferToPost_(deferToPost) { oops::Log::trace() << "ObsProcessorBase constructor" << std::endl; - data_.associate(flags_, "QCflagsData"); - data_.associate(obserr_, "ObsErrorData"); + ASSERT(flags); + ASSERT(obserr); + data_.associate(*flags_, "QCflagsData"); + data_.associate(*obserr_, "ObsErrorData"); } // ----------------------------------------------------------------------------- diff --git a/src/ufo/filters/ObsProcessorBase.h b/src/ufo/filters/ObsProcessorBase.h index 52dd80c85..9c32b3c5c 100644 --- a/src/ufo/filters/ObsProcessorBase.h +++ b/src/ufo/filters/ObsProcessorBase.h @@ -35,8 +35,8 @@ namespace ufo { class ObsProcessorBase : public ObsFilterBase { public: ObsProcessorBase(ioda::ObsSpace &, bool deferToPost, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ObsProcessorBase(); void preProcess() override; @@ -54,8 +54,8 @@ class ObsProcessorBase : public ObsFilterBase { protected: ioda::ObsSpace & obsdb_; - ioda::ObsDataVector & flags_; - ioda::ObsDataVector & obserr_; + std::shared_ptr> flags_; + std::shared_ptr> obserr_; ufo::Variables allvars_; ObsFilterData data_; bool prior_; diff --git a/src/ufo/filters/ObsRefractivityGradientCheck.cc b/src/ufo/filters/ObsRefractivityGradientCheck.cc index 210a265f4..7e9b576dc 100644 --- a/src/ufo/filters/ObsRefractivityGradientCheck.cc +++ b/src/ufo/filters/ObsRefractivityGradientCheck.cc @@ -26,8 +26,8 @@ namespace ufo { ObsRefractivityGradientCheck::ObsRefractivityGradientCheck( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "ObsRefractivityGradientCheck: " @@ -91,7 +91,7 @@ void ObsRefractivityGradientCheck::applyFilter( for (size_t jv = 0; jv < filtervars.nvars(); ++jv) { const size_t iv = observed.find(filtervars.variable(jv).variable()); for (size_t isort : idx) { - if (apply[obs_numbers[isort]] && flags_[iv][obs_numbers[isort]] == QCflags::pass) { + if ( apply[obs_numbers[isort]] && (*flags_)[iv][obs_numbers[isort]] == QCflags::pass ) { if ((gradient[isort] <= parameters_.gradientMin.value() || gradient[isort] >= parameters_.gradientMax.value() || gradient[isort] == 0 || @@ -100,7 +100,7 @@ void ObsRefractivityGradientCheck::applyFilter( height[obs_numbers[isort]] < parameters_.maxCheckHeight.value()) { // reject all observations below isort for (size_t jobs = isort; jobs < idx.size(); ++jobs) { - if (apply[obs_numbers[jobs]] && flags_[iv][obs_numbers[jobs]] == QCflags::pass) { + if (apply[obs_numbers[jobs]] && (*flags_)[iv][obs_numbers[jobs]] == QCflags::pass) { flagged[jv][obs_numbers[jobs]] = true; } } diff --git a/src/ufo/filters/ObsRefractivityGradientCheck.h b/src/ufo/filters/ObsRefractivityGradientCheck.h index d19201a43..bd4eaf1d5 100644 --- a/src/ufo/filters/ObsRefractivityGradientCheck.h +++ b/src/ufo/filters/ObsRefractivityGradientCheck.h @@ -59,8 +59,8 @@ class ObsRefractivityGradientCheck : public FilterBase, static const std::string classname() {return "ufo::ObsRefractivityGradientCheck";} ObsRefractivityGradientCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ObsRefractivityGradientCheck(); private: diff --git a/src/ufo/filters/OceanVerticalStabilityCheck.cc b/src/ufo/filters/OceanVerticalStabilityCheck.cc index ed05e8961..d05fbe41c 100644 --- a/src/ufo/filters/OceanVerticalStabilityCheck.cc +++ b/src/ufo/filters/OceanVerticalStabilityCheck.cc @@ -22,8 +22,8 @@ namespace ufo { OceanVerticalStabilityCheck::OceanVerticalStabilityCheck( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "OceanVerticalStabilityCheck constructor" << std::endl; @@ -90,7 +90,7 @@ void OceanVerticalStabilityCheck::applyFilter(const std::vector & apply, // if any filter variable fails QC): const std::vector obs_indices = obsAccessor.getValidObsIdsInProfile(iProfile, apply, - flags_, + *flags_, filtervars, false); diff --git a/src/ufo/filters/OceanVerticalStabilityCheck.h b/src/ufo/filters/OceanVerticalStabilityCheck.h index 66fef178b..c886a0783 100644 --- a/src/ufo/filters/OceanVerticalStabilityCheck.h +++ b/src/ufo/filters/OceanVerticalStabilityCheck.h @@ -103,8 +103,8 @@ class OceanVerticalStabilityCheck : public FilterBase, static const std::string classname() {return "ufo::OceanVerticalStabilityCheck";} OceanVerticalStabilityCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); ~OceanVerticalStabilityCheck(); private: diff --git a/src/ufo/filters/ParameterSubstitution.cc b/src/ufo/filters/ParameterSubstitution.cc index 85ef36d8c..e2670d4ba 100644 --- a/src/ufo/filters/ParameterSubstitution.cc +++ b/src/ufo/filters/ParameterSubstitution.cc @@ -21,8 +21,8 @@ namespace ufo { ParameterSubstitution::ParameterSubstitution(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : parameters_(parameters) { oops::Log::trace() << "ParameterSubstitution constructor" << std::endl; diff --git a/src/ufo/filters/ParameterSubstitution.h b/src/ufo/filters/ParameterSubstitution.h index 5d456933a..ebf68165e 100644 --- a/src/ufo/filters/ParameterSubstitution.h +++ b/src/ufo/filters/ParameterSubstitution.h @@ -110,8 +110,8 @@ class ParameterSubstitution : public ObsFilterBase, ParameterSubstitution(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ParameterSubstitution() {} diff --git a/src/ufo/filters/PerformAction.cc b/src/ufo/filters/PerformAction.cc index 0aed1a4af..d27c657e1 100644 --- a/src/ufo/filters/PerformAction.cc +++ b/src/ufo/filters/PerformAction.cc @@ -50,8 +50,8 @@ std::vector> PerformActionParameters // ----------------------------------------------------------------------------- PerformAction::PerformAction(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "PerformAction constructor" << std::endl; diff --git a/src/ufo/filters/PerformAction.h b/src/ufo/filters/PerformAction.h index ba33289e8..b19a6984e 100644 --- a/src/ufo/filters/PerformAction.h +++ b/src/ufo/filters/PerformAction.h @@ -65,8 +65,8 @@ class PerformAction : public FilterBase, static const std::string classname() {return "ufo::PerformAction";} PerformAction(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); private: void print(std::ostream &) const override; diff --git a/src/ufo/filters/PoissonDiskThinning.cc b/src/ufo/filters/PoissonDiskThinning.cc index 98d7cb7dd..33ba79fe0 100644 --- a/src/ufo/filters/PoissonDiskThinning.cc +++ b/src/ufo/filters/PoissonDiskThinning.cc @@ -166,8 +166,8 @@ struct PoissonDiskThinning::ObsData PoissonDiskThinning::PoissonDiskThinning(ioda::ObsSpace & obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), options_(parameters) { oops::Log::trace() << "PoissonDiskThinning constructor" << std::endl; oops::Log::debug() << "PoissonDiskThinning: config = " << options_ << std::endl; @@ -413,7 +413,7 @@ std::vector PoissonDiskThinning::getValidObservationIds( const std::vector & apply, const Variables & filtervars, const ObsAccessor &obsAccessor) const { - std::vector validObsIds = obsAccessor.getValidObservationIds(apply, flags_, filtervars); + std::vector validObsIds = obsAccessor.getValidObservationIds(apply, *flags_, filtervars); if (!options_.shuffle) { // The user wants to process observations in fixed (non-random) order. Ensure the filter @@ -717,14 +717,14 @@ void PoissonDiskThinning::thinCategoryMedian(const ObsData &obsData, template std::array PoissonDiskThinning::getObservationPosition( size_t obsId, const ObsData &obsData) const { - std::array position = {util::missingValue()}; + std::array position; unsigned int dim = 0; if (obsData.latitudes && obsData.longitudes) { if (obsData.minLatitudeSpacings && obsData.minLongitudeSpacings) { - position.at(dim++) = (*obsData.latitudes)[obsId]; - position.at(dim++) = (*obsData.longitudes)[obsId]; + position[dim++] = (*obsData.latitudes)[obsId]; + position[dim++] = (*obsData.longitudes)[obsId]; } else { const float deg2rad = static_cast(M_PI / 180.0); const float earthRadius = static_cast(Constants::mean_earth_rad_m / 1000.0); @@ -736,20 +736,20 @@ std::array PoissonDiskThinning::getObservationPosition( const float sinLon = std::sin(lon); const float cosLon = std::cos(lon); - position.at(dim++) = earthRadius * cosLat * cosLon; - position.at(dim++) = earthRadius * cosLat * sinLon; - position.at(dim++) = earthRadius * sinLat; + position[dim++] = earthRadius * cosLat * cosLon; + position[dim++] = earthRadius * cosLat * sinLon; + position[dim++] = earthRadius * sinLat; } } if (obsData.pressures) { - position.at(dim++) = (*obsData.pressures)[obsId]; + position[dim++] = (*obsData.pressures)[obsId]; } if (obsData.times) { // We use the centre of the assimilation window as the reference time when converting // datetimes to floats. - position.at(dim++) = ((*obsData.times)[obsId] - obsdb_.timeWindow().midpoint()).toSeconds(); + position[dim++] = ((*obsData.times)[obsId] - obsdb_.timeWindow().midpoint()).toSeconds(); } return position; @@ -759,7 +759,7 @@ template std::array PoissonDiskThinning::getExclusionVolumeSemiAxes( size_t obsId, const ObsData &obsData) const { - std::array semiAxes = {util::missingValue()}; + std::array semiAxes; const int priority = obsData.priorities == boost::none ? 0 : (*obsData.priorities)[obsId]; @@ -773,20 +773,20 @@ std::array PoissonDiskThinning::getExclusionVolumeSemiAxes( const float minEuclideanDistance = earthDiameter * std::sin(minGeodesicDistance * invEarthDiameter); - semiAxes.at(dim++) = minEuclideanDistance; - semiAxes.at(dim++) = minEuclideanDistance; - semiAxes.at(dim++) = minEuclideanDistance; + semiAxes[dim++] = minEuclideanDistance; + semiAxes[dim++] = minEuclideanDistance; + semiAxes[dim++] = minEuclideanDistance; } else if (obsData.minLatitudeSpacings && obsData.minLongitudeSpacings) { - semiAxes.at(dim++) = obsData.minLatitudeSpacings->at(priority); - semiAxes.at(dim++) = obsData.minLongitudeSpacings->at(priority); + semiAxes[dim++] = obsData.minLatitudeSpacings->at(priority); + semiAxes[dim++] = obsData.minLongitudeSpacings->at(priority); } if (obsData.minVerticalSpacings) { - semiAxes.at(dim++) = obsData.minVerticalSpacings->at(priority); + semiAxes[dim++] = obsData.minVerticalSpacings->at(priority); } if (obsData.minTimeSpacings) { - semiAxes.at(dim++) = obsData.minTimeSpacings->at(priority).toSeconds(); + semiAxes[dim++] = obsData.minTimeSpacings->at(priority).toSeconds(); } return semiAxes; diff --git a/src/ufo/filters/PoissonDiskThinning.h b/src/ufo/filters/PoissonDiskThinning.h index 10a333bdd..0fbd92ae1 100644 --- a/src/ufo/filters/PoissonDiskThinning.h +++ b/src/ufo/filters/PoissonDiskThinning.h @@ -52,8 +52,8 @@ class PoissonDiskThinning : public FilterBase, static const std::string classname() {return "ufo::PoissonDiskThinning";} PoissonDiskThinning(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); ~PoissonDiskThinning() override; diff --git a/src/ufo/filters/PreQC.cc b/src/ufo/filters/PreQC.cc index c5891a40f..67840003a 100644 --- a/src/ufo/filters/PreQC.cc +++ b/src/ufo/filters/PreQC.cc @@ -17,8 +17,8 @@ namespace ufo { PreQC::PreQC(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "PreQC constructor" << std::endl; diff --git a/src/ufo/filters/PreQC.h b/src/ufo/filters/PreQC.h index c69de8d77..27a116a02 100644 --- a/src/ufo/filters/PreQC.h +++ b/src/ufo/filters/PreQC.h @@ -45,8 +45,8 @@ class PreQC : public FilterBase, private util::ObjectCounter { static const std::string classname() {return "ufo::PreQC";} PreQC(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); private: void print(std::ostream &) const override; diff --git a/src/ufo/filters/PrintFilterData.cc b/src/ufo/filters/PrintFilterData.cc index 5575faf49..d2fa4b202 100644 --- a/src/ufo/filters/PrintFilterData.cc +++ b/src/ufo/filters/PrintFilterData.cc @@ -22,9 +22,9 @@ namespace ufo { PrintFilterData::PrintFilterData(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) - : ObsProcessorBase(obsdb, parameters.deferToPost, flags, obserr), + std::shared_ptr > flags, + std::shared_ptr > obserr) + : ObsProcessorBase(obsdb, parameters.deferToPost, std::move(flags), std::move(obserr)), parameters_(parameters), os_(parameters.outputToTest ? oops::Log::test() : oops::Log::info()) { diff --git a/src/ufo/filters/PrintFilterData.h b/src/ufo/filters/PrintFilterData.h index 1308b6425..82ed81090 100644 --- a/src/ufo/filters/PrintFilterData.h +++ b/src/ufo/filters/PrintFilterData.h @@ -149,8 +149,8 @@ class PrintFilterData : public ObsProcessorBase, static const std::string classname() {return "ufo::PrintFilterData";} PrintFilterData(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); private: // variables Parameters_ parameters_; diff --git a/src/ufo/filters/ProbabilityGrossErrorWholeReport.cc b/src/ufo/filters/ProbabilityGrossErrorWholeReport.cc index f543b0c7e..769b3753a 100644 --- a/src/ufo/filters/ProbabilityGrossErrorWholeReport.cc +++ b/src/ufo/filters/ProbabilityGrossErrorWholeReport.cc @@ -23,8 +23,8 @@ namespace ufo { // ----------------------------------------------------------------------------- ProbabilityGrossErrorWholeReport::ProbabilityGrossErrorWholeReport(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { diff --git a/src/ufo/filters/ProbabilityGrossErrorWholeReport.h b/src/ufo/filters/ProbabilityGrossErrorWholeReport.h index efb0a4b6b..403b1d6d4 100644 --- a/src/ufo/filters/ProbabilityGrossErrorWholeReport.h +++ b/src/ufo/filters/ProbabilityGrossErrorWholeReport.h @@ -103,8 +103,8 @@ class ProbabilityGrossErrorWholeReport : public FilterBase, static const std::string classname() {return "ufo::ProbabilityGrossErrorWholeReport";} ProbabilityGrossErrorWholeReport(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr>, + std::shared_ptr>); ~ProbabilityGrossErrorWholeReport(); private: diff --git a/src/ufo/filters/ProcessAMVQI.cc b/src/ufo/filters/ProcessAMVQI.cc index 7b1c0b0ff..2081f8f5c 100644 --- a/src/ufo/filters/ProcessAMVQI.cc +++ b/src/ufo/filters/ProcessAMVQI.cc @@ -24,8 +24,8 @@ namespace ufo { // ----------------------------------------------------------------------------- ProcessAMVQI::ProcessAMVQI(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr> flags, + std::shared_ptr> obserr) : ObsProcessorBase(obsdb, false /*deferToPost?*/, flags, obserr), parameters_(parameters) { diff --git a/src/ufo/filters/ProcessAMVQI.h b/src/ufo/filters/ProcessAMVQI.h index b42277f3d..337c381b6 100644 --- a/src/ufo/filters/ProcessAMVQI.h +++ b/src/ufo/filters/ProcessAMVQI.h @@ -62,8 +62,8 @@ class ProcessAMVQI : public ObsProcessorBase, static const std::string classname() {return "ufo::ProcessAMVQI";} ProcessAMVQI(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr> flags, + std::shared_ptr> obserr); ~ProcessAMVQI() override; private: diff --git a/src/ufo/filters/ProfileAverageObsToModLevels.cc b/src/ufo/filters/ProfileAverageObsToModLevels.cc index a241426a7..9d64faa10 100644 --- a/src/ufo/filters/ProfileAverageObsToModLevels.cc +++ b/src/ufo/filters/ProfileAverageObsToModLevels.cc @@ -30,8 +30,8 @@ namespace ufo { ProfileAverageObsToModLevels::ProfileAverageObsToModLevels( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "ProfileAverageObsToModLevels constructor" << std::endl; @@ -64,7 +64,7 @@ void ProfileAverageObsToModLevels::applyFilter(const std::vector & apply, const Variables & filtervars, std::vector> & flagged) const { oops::Log::trace() << "ProfileAverageObsToModLevels applyFilter start" << std::endl; - oops::Log::debug() << "ProfileAverageObsToModLevels obserr: " << obserr_ << std::endl; + oops::Log::debug() << "ProfileAverageObsToModLevels obserr: " << *obserr_ << std::endl; const oops::ObsVariables observed = obsdb_.obsvariables(); ioda::ObsDataVector obs(obsdb_, filtervars.toOopsObsVariables(), "ObsValue"); @@ -121,14 +121,14 @@ void ProfileAverageObsToModLevels::applyFilter(const std::vector & apply, ufo::setModLevelFlags(locsExt, link_obs_and_mod_inds, flagged[jv], - (flags_)[iv]); + (*flags_)[iv]); // Compute increments at each obs level, take their average for each model level, // and add the average to the HofX at that model level: ufo::averageObsToModLevels(locsOriginal, locsExt, link_obs_and_mod_inds, apply, - flags_[iv], + (*flags_)[iv], hofx, obs[jv]); } // profile jprof diff --git a/src/ufo/filters/ProfileAverageObsToModLevels.h b/src/ufo/filters/ProfileAverageObsToModLevels.h index 059c0008c..cdf50287e 100644 --- a/src/ufo/filters/ProfileAverageObsToModLevels.h +++ b/src/ufo/filters/ProfileAverageObsToModLevels.h @@ -83,8 +83,8 @@ class ProfileAverageObsToModLevels : public FilterBase, static const std::string classname() {return "ufo::ProfileAverageObsToModLevels";} ProfileAverageObsToModLevels(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ProfileAverageObsToModLevels(); private: diff --git a/src/ufo/filters/ProfileBackgroundCheck.cc b/src/ufo/filters/ProfileBackgroundCheck.cc index 29f9679ad..9fb45a6c2 100644 --- a/src/ufo/filters/ProfileBackgroundCheck.cc +++ b/src/ufo/filters/ProfileBackgroundCheck.cc @@ -46,8 +46,8 @@ namespace ufo { ProfileBackgroundCheck::ProfileBackgroundCheck( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "ProfileBackgroundCheck constructor" << std::endl; @@ -71,7 +71,7 @@ void ProfileBackgroundCheck::applyFilter(const std::vector & apply, oops::Log::trace() << "ProfileBackgroundCheck applyFilter start" << std::endl; const oops::ObsVariables observed = obsdb_.obsvariables(); - oops::Log::debug() << "ProfileBackgroundCheck obserr: " << obserr_; + oops::Log::debug() << "ProfileBackgroundCheck obserr: " << *obserr_; ioda::ObsDataVector obs(obsdb_, filtervars.toOopsObsVariables(), "ObsValue"); ioda::ObsDataVector bias(obsdb_, filtervars.toOopsObsVariables(), "ObsBias", false); @@ -114,11 +114,11 @@ void ProfileBackgroundCheck::applyFilter(const std::vector & apply, // Determine whether this profile uses threshold or abs_threshold based // on the first value in the profile. for (size_t jobs : obs_numbers) { - if (apply[jobs] && flags_[iv][jobs] == QCflags::pass) { + if (apply[jobs] && (*flags_)[iv][jobs] == QCflags::pass) { // style note: use missingValue because obserr_ is declared in another file // and its underlying type could in principle change without it being obvious here. - ASSERT(obserr_[iv][jobs] != - util::missingValue>()); + ASSERT((*obserr_)[iv][jobs] != + util::missingValue>()); ASSERT(obs[jv][jobs] != util::missingValue()); ASSERT(hofx[jobs] != util::missingValue()); @@ -139,11 +139,11 @@ void ProfileBackgroundCheck::applyFilter(const std::vector & apply, const float yy = obs[jv][jobs] - bias[jv][jobs]; // Accumulate distance from background - if (obserr_[iv][jobs] > 0) { + if ((*obserr_)[iv][jobs] > 0) { total_nobs++; if (using_rel_thresh) { // Normalise by observation error - total_diff += std::pow((hofx[jobs] - yy) / obserr_[iv][jobs], 2); + total_diff += std::pow((hofx[jobs] - yy) / (*obserr_)[iv][jobs], 2); } else { // Use the absolute differences total_diff += std::pow(hofx[jobs] - yy, 2); @@ -161,7 +161,7 @@ void ProfileBackgroundCheck::applyFilter(const std::vector & apply, // Second run through the observations to apply the threshold to reject profiles for (size_t jobs : obs_numbers) { - if (apply[jobs] && flags_[iv][jobs] == QCflags::pass) { + if (apply[jobs] && (*flags_)[iv][jobs] == QCflags::pass) { // If rejecting this profile, then flag all elements if (total_diff > static_cast(profile_threshold)) flagged[jv][jobs] = true; } diff --git a/src/ufo/filters/ProfileBackgroundCheck.h b/src/ufo/filters/ProfileBackgroundCheck.h index c5c21059e..af7597cce 100644 --- a/src/ufo/filters/ProfileBackgroundCheck.h +++ b/src/ufo/filters/ProfileBackgroundCheck.h @@ -59,8 +59,8 @@ class ProfileBackgroundCheck : public FilterBase, static const std::string classname() {return "ufo::ProfileBackgroundCheck";} ProfileBackgroundCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ProfileBackgroundCheck(); private: diff --git a/src/ufo/filters/ProfileFewObsCheck.cc b/src/ufo/filters/ProfileFewObsCheck.cc index 88b3aea85..20d45634b 100644 --- a/src/ufo/filters/ProfileFewObsCheck.cc +++ b/src/ufo/filters/ProfileFewObsCheck.cc @@ -31,8 +31,8 @@ namespace ufo { ProfileFewObsCheck::ProfileFewObsCheck( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "ProfileFewObsCheck constructor" << std::endl; @@ -97,7 +97,7 @@ void ProfileFewObsCheck::applyFilter(const std::vector & apply, for (size_t jobs : obs_numbers) { if (apply[jobs]) { numTotal++; - if (flags_[jVar][jobs] == QCflags::pass) + if ((*flags_)[jVar][jobs] == QCflags::pass) numValid++; } } @@ -122,7 +122,7 @@ void ProfileFewObsCheck::applyFilter(const std::vector & apply, const size_t iFilterVar = iVar * nChans + iChan; const size_t jVar = variableIndicesMap[iChan]; for (size_t jobs : obs_numbers) - if (apply[jobs] && flags_[jVar][jobs] == QCflags::pass) + if (apply[jobs] && (*flags_)[jVar][jobs] == QCflags::pass) flagged[iFilterVar][jobs] = true; } } diff --git a/src/ufo/filters/ProfileFewObsCheck.h b/src/ufo/filters/ProfileFewObsCheck.h index 95d9c36d4..a61f18d55 100644 --- a/src/ufo/filters/ProfileFewObsCheck.h +++ b/src/ufo/filters/ProfileFewObsCheck.h @@ -52,8 +52,8 @@ class ProfileFewObsCheck : public FilterBase, static const std::string classname() {return "ufo::ProfileFewObsCheck";} ProfileFewObsCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ProfileFewObsCheck(); private: diff --git a/src/ufo/filters/ProfileMaxDifferenceCheck.cc b/src/ufo/filters/ProfileMaxDifferenceCheck.cc index bc5175b83..2a4f370bb 100644 --- a/src/ufo/filters/ProfileMaxDifferenceCheck.cc +++ b/src/ufo/filters/ProfileMaxDifferenceCheck.cc @@ -26,8 +26,8 @@ namespace ufo { ProfileMaxDifferenceCheck::ProfileMaxDifferenceCheck( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "ProfileMaxDifferenceCheck: " diff --git a/src/ufo/filters/ProfileMaxDifferenceCheck.h b/src/ufo/filters/ProfileMaxDifferenceCheck.h index 9629fffb1..01d1a1ba8 100644 --- a/src/ufo/filters/ProfileMaxDifferenceCheck.h +++ b/src/ufo/filters/ProfileMaxDifferenceCheck.h @@ -61,8 +61,8 @@ class ProfileMaxDifferenceCheck : public FilterBase, static const std::string classname() {return "ufo::ProfileMaxDifferenceCheck";} ProfileMaxDifferenceCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ProfileMaxDifferenceCheck(); private: diff --git a/src/ufo/filters/ProfileUnFlagObsCheck.cc b/src/ufo/filters/ProfileUnFlagObsCheck.cc index 8dbdb09d7..e8f9ae3cf 100644 --- a/src/ufo/filters/ProfileUnFlagObsCheck.cc +++ b/src/ufo/filters/ProfileUnFlagObsCheck.cc @@ -77,7 +77,7 @@ class AbsoluteToleranceCreator { static std::unique_ptr create_unique(std::string name, const ProfileUnFlagObsCheck::Parameters_& parameters, const ioda::ObsSpace& obsdb) { std::unique_ptr absoluteTolerance = - getFactory()[name]->create_unique(parameters, obsdb); + std::move(getFactory()[name]->create_unique(parameters, obsdb)); return absoluteTolerance; } static std::map &getFactory() { @@ -124,8 +124,8 @@ static AbsoluteTolerancePiecewiseCreator absoluteTolerancePiecewiseCreator; ProfileUnFlagObsCheck::ProfileUnFlagObsCheck( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "ProfileUnFlagObsCheck constructor" << std::endl; @@ -201,21 +201,21 @@ void ProfileUnFlagObsCheck::applyFilter(const std::vector & apply, for (size_t jObsProfile = 0; jObsProfile < obsNumbers.size(); ++jObsProfile) { size_t jObsGlobal = obsNumbers[jObsProfile]; const float absTol = (*absoluteToleranceCalculate)(jObsGlobal); - if ((flags_[jVar][jObsGlobal] != QCflags::pass) && - (flags_[jVar][jObsGlobal] != QCflags::passive) && - (flags_[jVar][jObsGlobal] != QCflags::missing)) { + if (((*flags_)[jVar][jObsGlobal] != QCflags::pass) && + ((*flags_)[jVar][jObsGlobal] != QCflags::passive) && + ((*flags_)[jVar][jObsGlobal] != QCflags::missing)) { bool makeValid = false; if (jObsProfile == 0) { makeValid = true; } else { const size_t jObsMinus = obsNumbers[jObsProfile-1]; - makeValid = flags_[jVar][jObsMinus] == QCflags::pass && + makeValid = (*flags_)[jVar][jObsMinus] == QCflags::pass && (std::abs(variableData[jObsMinus] - variableData[jObsGlobal]) < absTol); } if (jObsProfile < obsNumbers.size()-1) { const size_t jObsPlus = obsNumbers[jObsProfile+1]; makeValid = makeValid && - (flags_[jVar][jObsPlus] == QCflags::pass) && + ((*flags_)[jVar][jObsPlus] == QCflags::pass) && (std::abs(variableData[jObsPlus] - variableData[jObsGlobal]) < absTol); } if (makeValid) { diff --git a/src/ufo/filters/ProfileUnFlagObsCheck.h b/src/ufo/filters/ProfileUnFlagObsCheck.h index 6d7934844..f5062167b 100644 --- a/src/ufo/filters/ProfileUnFlagObsCheck.h +++ b/src/ufo/filters/ProfileUnFlagObsCheck.h @@ -91,8 +91,8 @@ class ProfileUnFlagObsCheck : public FilterBase, static const std::string classname() {return "ufo::ProfileUnFlagObsCheck";} ProfileUnFlagObsCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ProfileUnFlagObsCheck(); private: diff --git a/src/ufo/filters/QCmanager.cc b/src/ufo/filters/QCmanager.cc index c8a37b0b7..f75d3eac4 100644 --- a/src/ufo/filters/QCmanager.cc +++ b/src/ufo/filters/QCmanager.cc @@ -47,33 +47,35 @@ void updateQCFlags(const std::vector *obsValues, std::vector& qcflag // ----------------------------------------------------------------------------- QCmanager::QCmanager(ioda::ObsSpace & obsdb, const Parameters_ & /*parameters*/, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & /*obserr*/) - : obsdb_(obsdb), nogeovals_(), nodiags_(), flags_(flags) + std::shared_ptr > qcflags, + std::shared_ptr > /*obserr*/) + : obsdb_(obsdb), nogeovals_(), nodiags_(), flags_(qcflags) { oops::Log::trace() << "QCmanager constructor start" << std::endl; + ASSERT(qcflags); + const oops::ObsVariables &allObservedVars = obsdb.obsvariables(); const oops::ObsVariables &initialObservedVars = obsdb.initial_obsvariables(); const oops::ObsVariables &derivedObservedVars = obsdb.derived_obsvariables(); ASSERT(allObservedVars.size() == initialObservedVars.size() + derivedObservedVars.size()); - ASSERT(flags_.nvars() == allObservedVars.size()); - ASSERT(flags_.nlocs() == obsdb_.nlocs()); + ASSERT(flags_->nvars() == allObservedVars.size()); + ASSERT(flags_->nlocs() == obsdb_.nlocs()); const ioda::ObsDataVector obs(obsdb, initialObservedVars, "ObsValue"); // Iterate over initial observed variables for (size_t jv = 0; jv < initialObservedVars.size(); ++jv) { const ioda::ObsDataRow ¤tObsValues = obs[jv]; - ioda::ObsDataRow ¤tQCFlags = flags[obs.varnames()[jv]]; + ioda::ObsDataRow ¤tQCFlags = (*qcflags)[obs.varnames()[jv]]; updateQCFlags(¤tObsValues, currentQCFlags); } // Iterate over derived variables and if they don't exist yet, set their QC flags to // 'missing'. for (size_t jv = 0; jv < derivedObservedVars.size(); ++jv) { - ioda::ObsDataRow ¤tQCFlags = flags[derivedObservedVars[jv]]; + ioda::ObsDataRow ¤tQCFlags = (*qcflags)[derivedObservedVars[jv]]; if (!obsdb.has("ObsValue", derivedObservedVars[jv])) { updateQCFlags(nullptr, currentQCFlags); } else { @@ -101,8 +103,8 @@ void QCmanager::postFilter(const GeoVaLs &, /*geovals*/ for (size_t jv = 0; jv < allObservedVars.size(); ++jv) { for (size_t jobs = 0; jobs < obsdb_.nlocs(); ++jobs) { const size_t iobs = allObservedVars.size() * jobs + jv; - if (flags_[allObservedVars[jv]][jobs] == 0 && hofx[iobs] == missing) { - flags_[allObservedVars[jv]][jobs] = QCflags::Hfailed; + if ((*flags_)[allObservedVars[jv]][jobs] == 0 && hofx[iobs] == missing) { + (*flags_)[allObservedVars[jv]][jobs] = QCflags::Hfailed; } } } @@ -163,8 +165,9 @@ void QCmanager::print(std::ostream & os) const { const std::string varName = allObservedVars[jvar]; std::unique_ptr>> accumulator = obsdb_.distribution()->createAccumulator(cases.size()); + for (size_t jobs = 0; jobs < nlocs; ++jobs) { - const int actualFlag = flags_[jvar][jobs]; + const int actualFlag = (*flags_)[jvar][jobs]; for (size_t jcase = 0; jcase < cases.size(); ++jcase) if (actualFlag == cases[jcase].first) accumulator->addTerm(jobs, jcase, 1); @@ -223,7 +226,7 @@ void QCmanager::print(std::ostream & os) const { const std::vector countDiags = accumulatorDiags->computeResult(); if (obsdb_.comm().rank() == 0) { - const std::string info = "QC " + flags_.obstype() + " " + varName + ": "; + const std::string info = "QC " + flags_->obstype() + " " + varName + ": "; // Normal cases for (size_t i = numSpecialCases; i < counts.size(); ++i) diff --git a/src/ufo/filters/QCmanager.h b/src/ufo/filters/QCmanager.h index 4e7407afa..3bac7f509 100644 --- a/src/ufo/filters/QCmanager.h +++ b/src/ufo/filters/QCmanager.h @@ -41,8 +41,8 @@ class QCmanager : public ObsFilterBase { typedef QCmanagerParameters Parameters_; QCmanager(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~QCmanager(); void preProcess() override {} @@ -62,7 +62,7 @@ class QCmanager : public ObsFilterBase { ioda::ObsSpace & obsdb_; const oops::Variables nogeovals_; const oops::ObsVariables nodiags_; - ioda::ObsDataVector & flags_; + std::shared_ptr> flags_; }; } // namespace ufo diff --git a/src/ufo/filters/SatName.cc b/src/ufo/filters/SatName.cc index d5707d9a2..fff1550de 100644 --- a/src/ufo/filters/SatName.cc +++ b/src/ufo/filters/SatName.cc @@ -63,8 +63,8 @@ std::string get_sat_name(int SatID, const std::vector &Sat // ----------------------------------------------------------------------------- SatName::SatName(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "SatName constructor" << std::endl; diff --git a/src/ufo/filters/SatName.h b/src/ufo/filters/SatName.h index cc3ab42b9..a1ea0e2cf 100644 --- a/src/ufo/filters/SatName.h +++ b/src/ufo/filters/SatName.h @@ -78,8 +78,8 @@ class SatName : public FilterBase, typedef SatNameParameters Parameters_; static const std::string classname() {return "ufo::SatName";} SatName(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~SatName(); private: void print(std::ostream &) const override; diff --git a/src/ufo/filters/SatwindInversionCorrection.cc b/src/ufo/filters/SatwindInversionCorrection.cc index 0407a573a..cf0a09a44 100644 --- a/src/ufo/filters/SatwindInversionCorrection.cc +++ b/src/ufo/filters/SatwindInversionCorrection.cc @@ -22,9 +22,9 @@ namespace ufo { // ----------------------------------------------------------------------------- SatwindInversionCorrection::SatwindInversionCorrection(ioda::ObsSpace & obsdb, - const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + const Parameters_ & parameters, + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "SatwindInversion constructor" << std::endl; diff --git a/src/ufo/filters/SatwindInversionCorrection.h b/src/ufo/filters/SatwindInversionCorrection.h index e82646fe3..6bf69933f 100644 --- a/src/ufo/filters/SatwindInversionCorrection.h +++ b/src/ufo/filters/SatwindInversionCorrection.h @@ -73,8 +73,8 @@ class SatwindInversionCorrection : public FilterBase, static const std::string classname() {return "ufo::SatwindInversionCorrection";} SatwindInversionCorrection(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~SatwindInversionCorrection(); private: diff --git a/src/ufo/filters/SpikeAndStepCheck.cc b/src/ufo/filters/SpikeAndStepCheck.cc index 98e0cac65..340dcec0f 100644 --- a/src/ufo/filters/SpikeAndStepCheck.cc +++ b/src/ufo/filters/SpikeAndStepCheck.cc @@ -31,8 +31,8 @@ namespace ufo { SpikeAndStepCheck::SpikeAndStepCheck( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "SpikeAndStepCheck constructor" << std::endl; @@ -144,7 +144,7 @@ void SpikeAndStepCheck::applyFilter(const std::vector & apply, // if any filter variable fails QC): const std::vector obs_indices = obsAccessor.getValidObsIdsInProfile(iProfile, apply, - flags_, + *flags_, filtervars, false); // Struct of y, x, dy, dx, dy/dx: diff --git a/src/ufo/filters/SpikeAndStepCheck.h b/src/ufo/filters/SpikeAndStepCheck.h index da5c12fcf..a5911afd4 100644 --- a/src/ufo/filters/SpikeAndStepCheck.h +++ b/src/ufo/filters/SpikeAndStepCheck.h @@ -161,8 +161,8 @@ class SpikeAndStepCheck : public FilterBase, static const std::string classname() {return "ufo::SpikeAndStepCheck";} SpikeAndStepCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); ~SpikeAndStepCheck(); /// \brief Vectors of dependent and independent variables, their differences, and gradient diff --git a/src/ufo/filters/StuckCheck.cc b/src/ufo/filters/StuckCheck.cc index e714adfb2..6d98553a2 100644 --- a/src/ufo/filters/StuckCheck.cc +++ b/src/ufo/filters/StuckCheck.cc @@ -28,8 +28,8 @@ namespace ufo { StuckCheck::StuckCheck(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), options_(parameters) { oops::Log::trace() << "StuckCheck constructor" << std::endl; diff --git a/src/ufo/filters/StuckCheck.h b/src/ufo/filters/StuckCheck.h index 9602fe6cf..7d3ed1a8f 100644 --- a/src/ufo/filters/StuckCheck.h +++ b/src/ufo/filters/StuckCheck.h @@ -48,8 +48,8 @@ class StuckCheck: public FilterBase, static const std::string classname() {return "ufo::StuckCheck";} StuckCheck(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); ~StuckCheck() override; diff --git a/src/ufo/filters/SuperOb.cc b/src/ufo/filters/SuperOb.cc index 498c78802..13ee3856b 100644 --- a/src/ufo/filters/SuperOb.cc +++ b/src/ufo/filters/SuperOb.cc @@ -16,8 +16,8 @@ namespace ufo { SuperOb::SuperOb(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), options_(parameters) { oops::Log::trace() << "SuperOb constructor start" << std::endl; @@ -37,7 +37,7 @@ SuperOb::SuperOb(ioda::ObsSpace & obsdb, std::unique_ptr superOb = SuperObFactory::create(params.superObName, - data_, apply, filtervars_, flags, flagged); + data_, apply, filtervars_, *flags, flagged); if (superOb->requireHofX()) { allvars_ += Variables(filtervars_, "HofX"); @@ -60,7 +60,7 @@ void SuperOb::applyFilter(const std::vector & apply, std::unique_ptr superOb = SuperObFactory::create(params.superObName, - data_, apply, filtervars, flags_, flagged); + data_, apply, filtervars, *flags_, flagged); superOb->runAlgorithm(); oops::Log::trace() << "SuperOb applyFilter complete" << std::endl; diff --git a/src/ufo/filters/SuperOb.h b/src/ufo/filters/SuperOb.h index 1dadb1634..94f8c56d4 100644 --- a/src/ufo/filters/SuperOb.h +++ b/src/ufo/filters/SuperOb.h @@ -70,8 +70,8 @@ class SuperOb : public FilterBase, static const std::string classname() {return "ufo::SuperOb";} SuperOb(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~SuperOb(); private: diff --git a/src/ufo/filters/SuperRefractionCheckImpactParameter.cc b/src/ufo/filters/SuperRefractionCheckImpactParameter.cc index 68d7489da..ef76194f0 100644 --- a/src/ufo/filters/SuperRefractionCheckImpactParameter.cc +++ b/src/ufo/filters/SuperRefractionCheckImpactParameter.cc @@ -31,8 +31,8 @@ namespace ufo { SuperRefractionCheckImpactParameter::SuperRefractionCheckImpactParameter( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "SuperRefractionCheckImpactParameter constructor" << std::endl; @@ -164,7 +164,7 @@ void SuperRefractionCheckImpactParameter::applyFilter( } for (size_t iVar = 0; iVar < filtervars.nvars(); ++iVar) { for (size_t iobs : obs_numbers) { - if (apply[iobs] && flags_[iVar][iobs] == QCflags::pass && + if (apply[iobs] && (*flags_)[iVar][iobs] == QCflags::pass && impactParameterObs[0][iobs] <= impactParameterModel[kLevel] && kLevel > 0) { flagged[iVar][iobs] = true; @@ -175,7 +175,7 @@ void SuperRefractionCheckImpactParameter::applyFilter( } else { // profileCheck = false for (size_t iVar = 0; iVar < filtervars.nvars(); ++iVar) { for (size_t iobs : obs_numbers) { - if (apply[iobs] && flags_[iVar][iobs] == QCflags::pass) { + if (apply[iobs] && (*flags_)[iVar][iobs] == QCflags::pass) { std::vector refracProfile; std::vector heightProfile; // count the number of valid levels of model profiles for iobs diff --git a/src/ufo/filters/SuperRefractionCheckImpactParameter.h b/src/ufo/filters/SuperRefractionCheckImpactParameter.h index 1438cb174..055837748 100644 --- a/src/ufo/filters/SuperRefractionCheckImpactParameter.h +++ b/src/ufo/filters/SuperRefractionCheckImpactParameter.h @@ -53,8 +53,8 @@ class SuperRefractionCheckImpactParameter : public FilterBase, static const std::string classname() {return "ufo::SuperRefractionCheckImpactParameter";} SuperRefractionCheckImpactParameter(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~SuperRefractionCheckImpactParameter(); private: diff --git a/src/ufo/filters/SuperRefractionCheckNBAM.cc b/src/ufo/filters/SuperRefractionCheckNBAM.cc index 6af211021..cdf14f7ba 100644 --- a/src/ufo/filters/SuperRefractionCheckNBAM.cc +++ b/src/ufo/filters/SuperRefractionCheckNBAM.cc @@ -30,8 +30,8 @@ namespace ufo { SuperRefractionCheckNBAM::SuperRefractionCheckNBAM( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "SuperRefractionCheckNBAM constructor" << std::endl; diff --git a/src/ufo/filters/SuperRefractionCheckNBAM.h b/src/ufo/filters/SuperRefractionCheckNBAM.h index 7a9c651e6..84c520f98 100644 --- a/src/ufo/filters/SuperRefractionCheckNBAM.h +++ b/src/ufo/filters/SuperRefractionCheckNBAM.h @@ -65,8 +65,8 @@ class SuperRefractionCheckNBAM : public FilterBase, static std::string classname() {return std::string("ufo::SuperRefractionCheckNBAM");} SuperRefractionCheckNBAM(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~SuperRefractionCheckNBAM(); private: diff --git a/src/ufo/filters/TemporalThinning.cc b/src/ufo/filters/TemporalThinning.cc index 7aad4f2dc..a8404091e 100644 --- a/src/ufo/filters/TemporalThinning.cc +++ b/src/ufo/filters/TemporalThinning.cc @@ -304,8 +304,8 @@ typename TemporalThinner::ForwardValidObsIndexIterator TemporalThinner::findNear } // namespace TemporalThinning::TemporalThinning(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), options_(parameters) { oops::Log::trace() << "TemporalThinning constructor" << std::endl; @@ -328,7 +328,7 @@ void TemporalThinning::applyFilter(const std::vector & apply, // returns what it has been passed without modification. // The value of `retainOnlyIfAllFilterVariablesAreValid` is set to `false` // because that is the default value used in the `ObsAccessor` class. - const RecordHandler recordHandler(obsdb_, filtervars, flags_, false); + const RecordHandler recordHandler(obsdb_, filtervars, *flags_, false); // If records are treated as single obs and a category variable is also used, // ensure that there are no records with multiple values of the category variable. @@ -380,7 +380,7 @@ std::vector TemporalThinning::identifyThinnedObservations( const Variables & filtervars, const ObsAccessor &obsAccessor) const { const std::vector validObsIds - = obsAccessor.getValidObservationIds(apply, flags_, filtervars); + = obsAccessor.getValidObservationIds(apply, *flags_, filtervars); RecursiveSplitter splitter = obsAccessor.splitObservationsIntoIndependentGroups(validObsIds); diff --git a/src/ufo/filters/TemporalThinning.h b/src/ufo/filters/TemporalThinning.h index f0ff435d2..85ea5dc59 100644 --- a/src/ufo/filters/TemporalThinning.h +++ b/src/ufo/filters/TemporalThinning.h @@ -45,8 +45,8 @@ class TemporalThinning : public FilterBase, static const std::string classname() {return "ufo::TemporalThinning";} TemporalThinning(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); ~TemporalThinning() override; diff --git a/src/ufo/filters/Thinning.cc b/src/ufo/filters/Thinning.cc index 0c241d02f..6aa31d05a 100644 --- a/src/ufo/filters/Thinning.cc +++ b/src/ufo/filters/Thinning.cc @@ -21,8 +21,8 @@ namespace ufo { // ----------------------------------------------------------------------------- Thinning::Thinning(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "Thinning constructor" << std::endl; diff --git a/src/ufo/filters/Thinning.h b/src/ufo/filters/Thinning.h index ab3f1610f..f66ea915a 100644 --- a/src/ufo/filters/Thinning.h +++ b/src/ufo/filters/Thinning.h @@ -58,8 +58,8 @@ class Thinning : public FilterBase, static const std::string classname() {return "ufo::Thinning";} Thinning(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~Thinning(); private: diff --git a/src/ufo/filters/TrackCheck.cc b/src/ufo/filters/TrackCheck.cc index 6798ebae9..e812e85d4 100644 --- a/src/ufo/filters/TrackCheck.cc +++ b/src/ufo/filters/TrackCheck.cc @@ -100,8 +100,8 @@ float TrackCheck::TrackObservation::getFailedChecksFraction() { TrackCheck::TrackCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), options_(parameters) { oops::Log::trace() << "TrackCheck constructor" << std::endl; @@ -125,7 +125,7 @@ void TrackCheck::applyFilter(const std::vector & apply, const std::vector validObsIds = options_.ignoreExistingQCFlags ? obsAccessor.getValidObservationIds(apply) : - obsAccessor.getValidObservationIds(apply, flags_, filtervars); + obsAccessor.getValidObservationIds(apply, *flags_, filtervars); RecursiveSplitter splitter = obsAccessor.splitObservationsIntoIndependentGroups(validObsIds); TrackCheckUtils::sortTracksChronologically(validObsIds, obsAccessor, splitter); diff --git a/src/ufo/filters/TrackCheck.h b/src/ufo/filters/TrackCheck.h index 1bc4e86b9..86b191069 100644 --- a/src/ufo/filters/TrackCheck.h +++ b/src/ufo/filters/TrackCheck.h @@ -82,8 +82,8 @@ class TrackCheck : public FilterBase, static const std::string classname() { return "ufo::TrackCheck"; } TrackCheck(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); ~TrackCheck() override; diff --git a/src/ufo/filters/TrackCheckShip.cc b/src/ufo/filters/TrackCheckShip.cc index 0eda488a6..7880a9793 100644 --- a/src/ufo/filters/TrackCheckShip.cc +++ b/src/ufo/filters/TrackCheckShip.cc @@ -35,8 +35,8 @@ namespace ufo { TrackCheckShip::TrackCheckShip(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), options_(parameters) { oops::Log::trace() << "TrackCheckShip constructor" << std::endl; @@ -176,12 +176,12 @@ void TrackCheckShip::applyFilter(const std::vector & apply, // returns what it has been passed without modification. // The value of `retainOnlyIfAllFilterVariablesAreValid` is set to `false` // because that is the default value used in the `ObsAccessor` class. - const RecordHandler recordHandler(obsdb_, filtervars, flags_, false); + const RecordHandler recordHandler(obsdb_, filtervars, *flags_, false); const std::vector validObsIds = obsAccessor.getValidObservationIds(options_.recordsAreSingleObs ? recordHandler.changeApplyIfRecordsAreSingleObs(apply) : apply, - flags_, + *flags_, filtervars); RecursiveSplitter splitter = obsAccessor.splitObservationsIntoIndependentGroups(validObsIds); diff --git a/src/ufo/filters/TrackCheckShip.h b/src/ufo/filters/TrackCheckShip.h index f76255000..a1602c04f 100644 --- a/src/ufo/filters/TrackCheckShip.h +++ b/src/ufo/filters/TrackCheckShip.h @@ -63,8 +63,8 @@ class TrackCheckShip: public FilterBase, static const std::string classname() {return "ufo::TrackCheckShip";} TrackCheckShip(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); ~TrackCheckShip() override; diff --git a/src/ufo/filters/VariableAssignment.cc b/src/ufo/filters/VariableAssignment.cc index d64583fa5..7ff4c3ae7 100644 --- a/src/ufo/filters/VariableAssignment.cc +++ b/src/ufo/filters/VariableAssignment.cc @@ -492,9 +492,9 @@ void AssignmentParameters::deserialize(util::CompositePath &path, VariableAssignment::VariableAssignment(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) - : ObsProcessorBase(obsdb, parameters.deferToPost, flags, obserr), + std::shared_ptr > flags, + std::shared_ptr > obserr) + : ObsProcessorBase(obsdb, parameters.deferToPost, std::move(flags), std::move(obserr)), parameters_(parameters) { oops::Log::trace() << "VariableAssignment constructor" << std::endl; @@ -521,7 +521,7 @@ void VariableAssignment::doFilter() { for (const AssignmentParameters &assignment : parameters_.assignments.value()) { const ufo::Variable variable = getVariable(assignment); const ioda::ObsDtype dtype = getDataType(assignment.type, variable, obsdb_); - assignToVariable(variable, dtype, assignment, apply, data_, obsdb_, flags_); + assignToVariable(variable, dtype, assignment, apply, data_, obsdb_, *flags_); } oops::Log::trace() << "VariableAssignment doFilter complete" << std::endl; diff --git a/src/ufo/filters/VariableAssignment.h b/src/ufo/filters/VariableAssignment.h index 757465d0a..37236fb9e 100644 --- a/src/ufo/filters/VariableAssignment.h +++ b/src/ufo/filters/VariableAssignment.h @@ -163,8 +163,8 @@ class VariableAssignment : public ObsProcessorBase, static const std::string classname() {return "ufo::VariableAssignment";} VariableAssignment(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); private: void print(std::ostream &) const override; diff --git a/src/ufo/filters/VariableTransforms.cc b/src/ufo/filters/VariableTransforms.cc index 78153dbd1..e615464d7 100644 --- a/src/ufo/filters/VariableTransforms.cc +++ b/src/ufo/filters/VariableTransforms.cc @@ -31,8 +31,8 @@ namespace ufo { VariableTransforms::VariableTransforms( ioda::ObsSpace& obsdb, const eckit::Configuration& config, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr> flags, + std::shared_ptr> obserr) : FilterBase(obsdb, config, flags, obserr), parameters_() { oops::Log::trace() << "VariableTransforms constructor" << std::endl; diff --git a/src/ufo/filters/VariableTransforms.h b/src/ufo/filters/VariableTransforms.h index f16ead255..f76e28918 100644 --- a/src/ufo/filters/VariableTransforms.h +++ b/src/ufo/filters/VariableTransforms.h @@ -49,8 +49,8 @@ class VariableTransforms : public FilterBase, // This Constructor function initializes an instance of the // filter based on options specified in the YAML configuration file. VariableTransforms(ioda::ObsSpace &, const eckit::Configuration &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr>, + std::shared_ptr>); // Destructor ~VariableTransforms(); diff --git a/src/ufo/filters/Variables.cc b/src/ufo/filters/Variables.cc index 2e0c8c3aa..547073a04 100644 --- a/src/ufo/filters/Variables.cc +++ b/src/ufo/filters/Variables.cc @@ -53,7 +53,7 @@ Variables::Variables(const oops::ObsVariables & oopsvars) ASSERT(oopsvars.size() % channels.size() == 0); std::vector varnames(nvars); for (size_t joopsvar = 0, jvar = 0; - joopsvar < oopsvars.size() && jvar < nvars; ++joopsvar) { + joopsvar < oopsvars.size(), jvar < nvars; ++joopsvar) { const size_t pos = oopsvars[joopsvar].find_last_of('_'); const std::string varname = oopsvars[joopsvar].substr(0, pos); if (std::find(varnames.begin(), varnames.end(), varname) == varnames.end()) { diff --git a/src/ufo/filters/actions/SetFlag.cc b/src/ufo/filters/actions/SetFlag.cc index ff8af7cdf..088f028b7 100644 --- a/src/ufo/filters/actions/SetFlag.cc +++ b/src/ufo/filters/actions/SetFlag.cc @@ -63,7 +63,7 @@ void SetFlag::apply(const Variables &vars, typedef bool (*Predicate)(int); // Pointer to a function taking a QC flag and returning true if observations with this QC flag // should be ignored and false otherwise - Predicate isIgnored = nullptr; + Predicate isIgnored; switch (parameters_.ignore.value()) { case IgnoredObservations::NONE: isIgnored = &returnFalse; diff --git a/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.cc b/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.cc index ae4b16258..a771be761 100644 --- a/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.cc +++ b/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.cc @@ -29,8 +29,8 @@ namespace ufo { GNSSROOneDVarCheck::GNSSROOneDVarCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "GNSSROOneDVarCheck constructor" << std::endl; @@ -91,8 +91,8 @@ void GNSSROOneDVarCheck::applyFilter(const std::vector & apply, } // Save qc flags to database for retrieval in fortran - needed for channel selection - flags_.save("FortranQC"); // temporary measure as per ROobserror qc - obserr_.save("FortranERR"); // Pass latest errors to 1DVar + flags_->save("FortranQC"); // temporary measure as per ROobserror qc + obserr_->save("FortranERR"); // Pass latest errors to 1DVar // Pass it all to fortran ufo_gnssroonedvarcheck_apply_f90(key_, @@ -100,7 +100,7 @@ void GNSSROOneDVarCheck::applyFilter(const std::vector & apply, apply_char.size(), apply_char[0]); // Read qc flags from database - flags_.read("FortranQC"); // temporary measure as per ROobserror qc + flags_->read("FortranQC"); // temporary measure as per ROobserror qc oops::Log::trace() << "GNSSROOneDVarCheck applyFilter complete" << std::endl; } diff --git a/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.h b/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.h index 587d3889c..ec110567a 100644 --- a/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.h +++ b/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.h @@ -57,8 +57,8 @@ class GNSSROOneDVarCheck : public FilterBase, GNSSROOneDVarCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~GNSSROOneDVarCheck(); private: diff --git a/src/ufo/filters/gnssroonedvarcheck/ufo_gnssroonedvarcheck_mod.f90 b/src/ufo/filters/gnssroonedvarcheck/ufo_gnssroonedvarcheck_mod.f90 index 9d4319f39..9c278d1e4 100644 --- a/src/ufo/filters/gnssroonedvarcheck/ufo_gnssroonedvarcheck_mod.f90 +++ b/src/ufo/filters/gnssroonedvarcheck/ufo_gnssroonedvarcheck_mod.f90 @@ -148,9 +148,9 @@ subroutine ufo_gnssroonedvarcheck_create(self, obsspace, bmatrix_filename, & end do write(message, '(A,L1)') 'no super check = ', noSuperCheck call fckit_log % debug(message) - write(message, '(A,F16.6)') 'Dry refractivity constant = ', dryRefractivityConstant + write(message, '(A,L1)') 'Dry refractivity constant = ', dryRefractivityConstant call fckit_log % debug(message) - write(message, '(A,F16.6)') 'Wet refractivity constant = ', wetRefractivityConstant + write(message, '(A,L1)') 'Wet refractivity constant = ', wetRefractivityConstant call fckit_log % debug(message) end subroutine ufo_gnssroonedvarcheck_create @@ -238,11 +238,11 @@ subroutine ufo_gnssroonedvarcheck_apply(self, geovals, apply) ! ! Diagnostics to push back to the obs-space ! + integer, allocatable :: indices(:) ! The indices of the diagnostic elements to be updated integer, allocatable :: niter(:) ! Number of iterations required to converge real(kind_real), allocatable :: initial_cost(:) ! Initial cost-function value real(kind_real), allocatable :: final_cost(:) ! Final cost-function value real(kind_real), allocatable :: dfs_list(:) ! Degrees of freedom for signal - integer :: ind ! Get the obs-space information nobs = obsspace_get_nlocs(self % obsdb) @@ -426,13 +426,15 @@ subroutine ufo_gnssroonedvarcheck_apply(self, geovals, apply) end if ! Save the diagnostic information + allocate(indices(1:nobs_profile)) do ipoint = 0, nobs_profile-1 - ind = 1 + ((index_vals(start_point+ipoint) - 1) / nlevels) - niter(ind) = Ob % niter - initial_cost(ind) = O_Bdiff - final_cost(ind) = Ob % jcost - dfs_list(ind) = DFS + indices(ipoint+1) = 1 + ((index_vals(start_point+ipoint) - 1) / nlevels) end do + niter(indices) = Ob % niter + initial_cost(indices) = O_Bdiff + final_cost(indices) = Ob % jcost + dfs_list(indices) = DFS + deallocate(indices) call deallocate_singleob(Ob) end do diff --git a/src/ufo/filters/obsfunctions/CircularDifference.cc b/src/ufo/filters/obsfunctions/CircularDifference.cc deleted file mode 100644 index ac02ed1d1..000000000 --- a/src/ufo/filters/obsfunctions/CircularDifference.cc +++ /dev/null @@ -1,118 +0,0 @@ -/* - * (C) Crown copyright 2026, Met Office - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include "ufo/filters/obsfunctions/CircularDifference.h" - -#include - -#include "ioda/ObsDataVector.h" -#include "oops/util/Logger.h" -#include "oops/util/missingValues.h" -#include "ufo/filters/ObsFilterData.h" - -namespace ufo { - -// Register this ObsFunction with the factory -static ObsFunctionMaker - makerCircularDifference_("CircularDifference"); - -// ----------------------------------------------------------------------------- - -CircularDifference::CircularDifference(const eckit::LocalConfiguration &conf) - : invars_() { - oops::Log::trace() << "CircularDifference constructor" << std::endl; - // Validate and deserialize parameters - options_.validateAndDeserialize(conf); - - // Add required variables - invars_ += options_.variableStart.value(); - invars_ += options_.variableEnd.value(); - - oops::Log::debug() << "CircularDifference: configured with period=" - << options_.circularPeriod.value() - << ", signed=" << options_.computeSignedDifference.value() - << std::endl; -} - -// ----------------------------------------------------------------------------- - -CircularDifference::~CircularDifference() { - oops::Log::trace() << "CircularDifference destructor" << std::endl; -} - -// ----------------------------------------------------------------------------- - -void CircularDifference::compute(const ObsFilterData &in, - ioda::ObsDataVector &out) const { - oops::Log::trace() << "CircularDifference compute start" << std::endl; - const size_t nlocs = in.nlocs(); - const float missing = util::missingValue(); - const float period = options_.circularPeriod.value(); - const bool signedDiff = options_.computeSignedDifference.value(); - - // Validate period - if (period <= 0.0f) { - throw eckit::BadValue("Circular period must be positive", Here()); - } - - // Get input variable data - const Variable &varStart = options_.variableStart.value(); - const Variable &varEnd = options_.variableEnd.value(); - - ioda::ObsDataVector startData(in.obsspace(), - varStart.toOopsObsVariables()); - in.get(varStart, startData); - - ioda::ObsDataVector endData(in.obsspace(), - varEnd.toOopsObsVariables()); - in.get(varEnd, endData); - - // Compute circular difference for each variable and location - for (size_t ivar = 0; ivar < out.nvars(); ++ivar) { - for (size_t iloc = 0; iloc < nlocs; ++iloc) { - const float start = startData[ivar][iloc]; - const float end = endData[ivar][iloc]; - - // Check for missing values - if (start == missing || end == missing) { - out[ivar][iloc] = missing; - continue; - } - - // Compute raw difference - float diff = end - start; - - // Normalize difference to [-period/2, period/2) - // This gives the signed circular difference - // Uses the formula: ((b - a + half) % period) - half - const float half = period / 2.0f; - diff = std::fmod(diff + half, period); - if (diff < 0.0f) { - diff += period; // Ensure non-negative result from fmod (consistent with Python's behavior) - } - diff -= half; - - // Apply signed/unsigned option - if (signedDiff) { - out[ivar][iloc] = diff; - } else { - out[ivar][iloc] = std::abs(diff); - } - } - } - oops::Log::trace() << "CircularDifference compute complete" << std::endl; -} - -// ----------------------------------------------------------------------------- - -const ufo::Variables &CircularDifference::requiredVariables() const { - return invars_; -} - -// ----------------------------------------------------------------------------- - -} // namespace ufo diff --git a/src/ufo/filters/obsfunctions/CircularDifference.h b/src/ufo/filters/obsfunctions/CircularDifference.h deleted file mode 100644 index d255fdbd1..000000000 --- a/src/ufo/filters/obsfunctions/CircularDifference.h +++ /dev/null @@ -1,113 +0,0 @@ -/* - * (C) Crown copyright 2026, Met Office - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#ifndef UFO_FILTERS_OBSFUNCTIONS_CIRCULARDIFFERENCE_H_ -#define UFO_FILTERS_OBSFUNCTIONS_CIRCULARDIFFERENCE_H_ - -#include - -#include "oops/util/parameters/Parameter.h" -#include "oops/util/parameters/Parameters.h" -#include "oops/util/parameters/RequiredParameter.h" - -#include "ufo/filters/obsfunctions/ObsFunctionBase.h" -#include "ufo/filters/Variable.h" -#include "ufo/filters/Variables.h" -#include "ufo/utils/parameters/ParameterTraitsVariable.h" - -namespace ufo { - -/// \brief Parameters for the CircularDifference ObsFunction -class CircularDifferenceParameters : public oops::Parameters { - OOPS_CONCRETE_PARAMETERS(CircularDifferenceParameters, Parameters) - - public: - /// Start variable (value to subtract from in the difference calculation) - oops::RequiredParameter variableStart{"variable start", this}; - - /// End variable (value to subtract in the difference calculation) - oops::RequiredParameter variableEnd{"variable end", this}; - - /// Whether to return signed difference (true) or absolute difference (false) - /// Default is true (signed difference). - oops::Parameter computeSignedDifference{"signed", true, this}; - - /// Circular period (e.g., 360.0 for degrees, 2*pi for radians, 24 for hours - /// in a day, etc.) - oops::RequiredParameter circularPeriod{"circular period", this}; -}; - -/// \brief Compute minimum circular difference between two variables -/// -/// This ObsFunction computes the **minimum (shortest-path)** difference -/// between two values defined on a circular (cyclic) domain, accounting -/// for wrap-around at the circular period boundary. -/// -/// Consider two angles: For A (variable start) and B (variable end), the -/// circular difference is the shortest angular distance from A to B. -/// -/// If signed = true: -/// Returns a signed difference in the range [-period/2, period/2) -/// For angles, positive values indicate B is clockwise from A -/// For angles, negative values indicate B is counter-clockwise from A -/// -/// If signed = false: -/// Returns the absolute (unsigned) difference in the range [0, period/2] -/// -/// Example for wind directions (period = 360): -/// - A=10°, B=350°: signed=-20°, unsigned=20° -/// - A=350°, B=10°: signed=20°, unsigned=20° -/// - A=0°, B=180°: signed=-180°, unsigned=180° -/// -/// Example for days of the week (period = 7): -/// - A=1 (Monday), B=6 (Saturday): signed=-2, unsigned=2 -/// - A=3 (Wednesday), B=4 (Thursday): signed=1, unsigned=1 -/// -/// Usage example: -/// \code{.yaml} -/// obs function: -/// name: ObsFunction/CircularDifference -/// options: -/// variable start: ObsValue/windDirectionA -/// variable end: ObsValue/windDirectionB -/// signed: true -/// circular period: 360.0 -/// \endcode -/// -/// \note -/// This function always returns the minimum (shortest-path) circular -/// difference. Selecting the complementary (long-path) difference is -/// not currently supported, but may be added as a future option. -class CircularDifference : public ObsFunctionBase { - public: - static const std::string classname() { return "CircularDifference"; } - - /// Constructor takes configuration and validates parameters - explicit CircularDifference(const eckit::LocalConfiguration &); - - /// Destructor - ~CircularDifference(); - - /// Compute the circular difference - void compute(const ObsFilterData &, ioda::ObsDataVector &) const; - - /// Return the required variables for this ObsFunction - const ufo::Variables &requiredVariables() const; - - private: - /// Parameters for this ObsFunction - CircularDifferenceParameters options_; - - /// Required variables - ufo::Variables invars_; -}; - -// ----------------------------------------------------------------------------- - -} // namespace ufo - -#endif // UFO_FILTERS_OBSFUNCTIONS_CIRCULARDIFFERENCE_H_ diff --git a/src/ufo/filters/obsfunctions/CloudDetectMinResidualAVHRR.cc b/src/ufo/filters/obsfunctions/CloudDetectMinResidualAVHRR.cc index 235ff22a7..f74dff940 100644 --- a/src/ufo/filters/obsfunctions/CloudDetectMinResidualAVHRR.cc +++ b/src/ufo/filters/obsfunctions/CloudDetectMinResidualAVHRR.cc @@ -235,6 +235,7 @@ void CloudDetectMinResidualAVHRR::compute(const ObsFilterData & in, // Minimum Residual Method (MRM) for Cloud Detection: // Determine model level index of the cloud top (lcloud) + // Find pressure of the cloud top (cldprs) // Estimate cloud fraction (cldfrac) // output: out = 0 clear channel // out = 1 cloudy channel @@ -258,12 +259,8 @@ void CloudDetectMinResidualAVHRR::compute(const ObsFilterData & in, dtempf = dtempf_in[2]; } else if (snow) { dtempf = dtempf_in[3]; - } else if (mixed) { - dtempf = dtempf_in[4]; } else { - std::string errString = "The surface type is not defined for location: "; - oops::Log::error() << errString << iloc << std::endl; - throw eckit::BadValue(errString); + dtempf = dtempf_in[4]; } std::vector> dbt(nchans, std::vector(nlevs)); for (size_t ichan=0; ichan < nchans; ++ichan) { @@ -276,6 +273,7 @@ void CloudDetectMinResidualAVHRR::compute(const ObsFilterData & in, // Set initial cloud condition int lcloud = 0; float cldfrac = 0.0; + float cldprs = prsl[0][iloc] * 0.01; // convert from [Pa] to [hPa] // Loop through vertical layer from surface to model top for (size_t k = 0 ; k < nlevs ; ++k) { @@ -307,6 +305,7 @@ void CloudDetectMinResidualAVHRR::compute(const ObsFilterData & in, sum3 = sum; lcloud = k + 1; // array index + 1 -> model coordinate index cldfrac = cloudp; + cldprs = prsl[k][iloc] * 0.01; } } // end of vertical loop @@ -335,6 +334,7 @@ void CloudDetectMinResidualAVHRR::compute(const ObsFilterData & in, } if (fabs(sumx2) < FLT_MIN) sumx2 = copysign(1.0e-12, sumx2); dts = std::fabs(sumx / sumx2); + float dts_save = dts; if (std::abs(dts) > 1.0) { if (sea == false) { dts = std::min(dtempf, dts); diff --git a/src/ufo/filters/obsfunctions/CloudDetectMinResidualIR.cc b/src/ufo/filters/obsfunctions/CloudDetectMinResidualIR.cc index dc56c6f6f..58af4a778 100644 --- a/src/ufo/filters/obsfunctions/CloudDetectMinResidualIR.cc +++ b/src/ufo/filters/obsfunctions/CloudDetectMinResidualIR.cc @@ -247,14 +247,23 @@ void CloudDetectMinResidualIR::compute(const ObsFilterData & in, // Minimum Residual Method (MRM) for Cloud Detection: // Determine model level index of the cloud top (lcloud) + // Find pressure of the cloud top (cldprs) // Estimate cloud fraction (cldfrac) // output: out = 0 clear channel // out = 1 cloudy channel // out = 2 clear channel but too sensitive to surface condition + // Set vectors to hold cloud infomation from cloud detection (cab be part of the output) + // std::vector cloud_prsl(nlocs); + // std::vector cloud_frac(nlocs); + // std::vector cloud_lev(nlocs); // Loop through locations for (size_t iloc=0; iloc < nlocs; ++iloc) { + // Initialize at each location + // cloud_lev[iloc] = 0; + // cloud_prsl[iloc] = 0.0; + // cloud_frac[iloc] = 0.0; float sum = 0.0, sum2 = 0.0, sum3 = 0.0; float tmp = 0.0; float cloudp = 0.0; @@ -269,6 +278,7 @@ void CloudDetectMinResidualIR::compute(const ObsFilterData & in, // Set initial cloud condition int lcloud = 0; float cldfrac = 0.0; + float cldprs = prsl[0][iloc] * 0.01; // convert from [Pa] to [hPa] float sum_min = 1.e20; // Loop through vertical layer from surface to model top @@ -304,6 +314,7 @@ void CloudDetectMinResidualIR::compute(const ObsFilterData & in, sum_min = sum; lcloud = k + 1; // array index + 1 -> model coordinate index cldfrac = cloudp; + cldprs = prsl[k][iloc] * 0.01; } } // end of vertical loop @@ -332,6 +343,10 @@ void CloudDetectMinResidualIR::compute(const ObsFilterData & in, // Active channels if (out[ichan][iloc] < 1 && tao_cld > 0.02) out[ichan][iloc] = 1; } + // cloud infomation output at model level + // cloud_lev[iloc] = lcloud; + // cloud_prsl[iloc] = cldprs; + // cloud_frac[iloc] = cldfrac; } else { // If no clouds is detected, do sensivity to surface temperature check // Initialize at each location diff --git a/src/ufo/filters/obsfunctions/HydrometeorCheckAMSUA.cc b/src/ufo/filters/obsfunctions/HydrometeorCheckAMSUA.cc index 8fbb73a0c..09c834695 100644 --- a/src/ufo/filters/obsfunctions/HydrometeorCheckAMSUA.cc +++ b/src/ufo/filters/obsfunctions/HydrometeorCheckAMSUA.cc @@ -86,7 +86,7 @@ void HydrometeorCheckAMSUA::compute(const ObsFilterData & in, // Set channel index int ich238 = 0, ich314 = 1, ich503 = 2, ich528 = 3, ich536 = 4; - int ich544 = 5, ich890 = 14; + int ich544 = 5, ich549 = 6, ich890 = 14; // Get test groups from options const std::string &biastermgrp = options_.testBiasTerm.value(); @@ -184,7 +184,7 @@ void HydrometeorCheckAMSUA::compute(const ObsFilterData & in, // Set parameters float w1f6 = 1.0/10.0, w2f6 = 1.0/0.80; - float w2f4 = 1.0/1.80; + float w1f4 = 1.0/0.30, w2f4 = 1.0/1.80; std::vector> affected_channels(nchans, std::vector(nlocs)); // Loop over locations diff --git a/src/ufo/filters/obsfunctions/HydrometeorCheckAMSUAclr.cc b/src/ufo/filters/obsfunctions/HydrometeorCheckAMSUAclr.cc index 78bbe7c38..6e33a67a0 100644 --- a/src/ufo/filters/obsfunctions/HydrometeorCheckAMSUAclr.cc +++ b/src/ufo/filters/obsfunctions/HydrometeorCheckAMSUAclr.cc @@ -91,10 +91,10 @@ void HydrometeorCheckAMSUAclr::compute(const ObsFilterData & in, // Set channel index int ich238 = 0, ich314 = 1, ich503 = 2, ich528 = 3, ich536 = 4; - int ich544 = 5, ich890 = 14; + int ich544 = 5, ich549 = 6, ich890 = 14; if (inst == "atms") { ich238 = 0, ich314 = 1, ich503 = 2, ich528 = 4, ich536 = 5; - ich544 = 6, ich890 = 15; + ich544 = 6, ich549 = 7, ich890 = 15; } // Get test groups from options diff --git a/src/ufo/filters/obsfunctions/HydrometeorCheckATMS.cc b/src/ufo/filters/obsfunctions/HydrometeorCheckATMS.cc index 689d7464f..2745ce030 100644 --- a/src/ufo/filters/obsfunctions/HydrometeorCheckATMS.cc +++ b/src/ufo/filters/obsfunctions/HydrometeorCheckATMS.cc @@ -92,10 +92,10 @@ void HydrometeorCheckATMS::compute(const ObsFilterData & in, // Set channel index (AMSU-A like channels) int ich238 = 0, ich314 = 1, ich503 = 2, ich528 = 4, ich536 = 5; - int ich544 = 6, ich890 = 15; + int ich544 = 6, ich549 = 7, ich890 = 15; // Set channel index (MHS like channels) - int ich1650 = 16, ich1830e = 21; + int ich1650 = 16, ich1830a = 17, ich1830b = 18, ich1830c = 19, ich1830d = 20, ich1830e = 21; // Get test groups from options const std::string &biastermgrp = options_.testBiasTerm.value(); @@ -209,7 +209,7 @@ void HydrometeorCheckATMS::compute(const ObsFilterData & in, // Set parameters float w1f6 = 1.0/10.0, w2f6 = 1.0/0.80; - float w2f4 = 1.0/1.80; + float w1f4 = 1.0/0.30, w2f4 = 1.0/1.80; std::vector> affected_channels(nchans, std::vector(nlocs)); // Loop over locations diff --git a/src/ufo/filters/obsfunctions/ObsErrorBoundMW.cc b/src/ufo/filters/obsfunctions/ObsErrorBoundMW.cc index 92f66f63b..4deefafa7 100644 --- a/src/ufo/filters/obsfunctions/ObsErrorBoundMW.cc +++ b/src/ufo/filters/obsfunctions/ObsErrorBoundMW.cc @@ -133,13 +133,13 @@ void ObsErrorBoundMW::compute(const ObsFilterData & in, } // Set channel numbers - int ich536 = 0, ich890 = 0; + int ich238, ich314, ich503, ich528, ich536, ich544, ich549, ich890; if (inst == "amsua") { - ich536 = 5; - ich890 = 15; + ich238 = 1, ich314 = 2, ich503 = 3, ich528 = 4, ich536 = 5; + ich544 = 6, ich549 = 7, ich890 = 15; } else if (inst == "atms") { - ich536 = 6; - ich890 = 16; + ich238 = 1, ich314 = 2, ich503 = 3, ich528 = 5, ich536 = 6; + ich544 = 7, ich549 = 8, ich890 = 16; } float thresholdfactor = 3.0; if (options_.thresholdfactor.value() != boost::none) { @@ -222,6 +222,7 @@ void ObsErrorBoundMW::compute(const ObsFilterData & in, const std::vector obserr0 = options_.obserrOriginal.value().get(); for (size_t ichan = 0; ichan < nchans; ++ichan) { + int channel = ichan + 1; in.get(Variable(flaggrp+"/brightnessTemperature", channels_)[ichan], qcflagdata); in.get(Variable(errgrp+"/brightnessTemperature", channels_)[ichan], obserrdata); for (size_t iloc = 0; iloc < nlocs; ++iloc) { diff --git a/src/ufo/filters/obsfunctions/ObsErrorFactorPressureCheck.cc b/src/ufo/filters/obsfunctions/ObsErrorFactorPressureCheck.cc index be0a145d7..b8da4a7a5 100644 --- a/src/ufo/filters/obsfunctions/ObsErrorFactorPressureCheck.cc +++ b/src/ufo/filters/obsfunctions/ObsErrorFactorPressureCheck.cc @@ -203,7 +203,7 @@ void ObsErrorFactorPressureCheck::compute(const ObsFilterData & data, formulas::Formulation formulation = formulas::Formulation::Rogers; int iflag; - double sat_specific_humidity = 0.0; + double sat_specific_humidity; const float grav = Constants::grav; const float deg2rad = Constants::deg2rad; const float grav_equator = Constants::grav_equator; @@ -270,7 +270,6 @@ void ObsErrorFactorPressureCheck::compute(const ObsFilterData & data, } } - sfcchk = 0.0f; if (reported_height) { fact = 0.0f; if (obs_height[iloc]-dstn[iloc] > 10.0f) { diff --git a/src/ufo/filters/obsfunctions/ObsErrorFactorSituDependMW.cc b/src/ufo/filters/obsfunctions/ObsErrorFactorSituDependMW.cc index 63673694c..649cb7558 100644 --- a/src/ufo/filters/obsfunctions/ObsErrorFactorSituDependMW.cc +++ b/src/ufo/filters/obsfunctions/ObsErrorFactorSituDependMW.cc @@ -172,13 +172,13 @@ void ObsErrorFactorSituDependMW::compute(const ObsFilterData & in, in.get(Variable("GeoVaLs/water_area_fraction"), water_frac); // Set channel number - int ich536 = 0, ich890 = 0; + int ich238, ich314, ich503, ich528, ich536, ich544, ich549, ich890; if (inst == "amsua") { - ich536 = 5; - ich890 = 15; + ich238 = 1, ich314 = 2, ich503 = 3, ich528 = 4, ich536 = 5; + ich544 = 6, ich549 = 7, ich890 = 15; } else if (inst == "atms") { - ich536 = 6; - ich890 = 16; + ich238 = 1, ich314 = 2, ich503 = 3, ich528 = 5, ich536 = 6; + ich544 = 7, ich549 = 8, ich890 = 16; } // Get Original Observation Error from ObsFunction diff --git a/src/ufo/filters/obsfunctions/ObsErrorFactorSurfJacobianRad.cc b/src/ufo/filters/obsfunctions/ObsErrorFactorSurfJacobianRad.cc index 6cc521b44..e9dd92118 100644 --- a/src/ufo/filters/obsfunctions/ObsErrorFactorSurfJacobianRad.cc +++ b/src/ufo/filters/obsfunctions/ObsErrorFactorSurfJacobianRad.cc @@ -89,7 +89,7 @@ void ObsErrorFactorSurfJacobianRad::compute(const ObsFilterData & in, size_t nlocs = in.nlocs(); size_t nchans = channels_.size(); - size_t ich536 = 0, ich890 = 0; + size_t ich536, ich890; if (inst == "amsua") { ich536 = 5; ich890 = 15; diff --git a/src/ufo/filters/obsfunctions/ObsErrorFactorTopoRad.cc b/src/ufo/filters/obsfunctions/ObsErrorFactorTopoRad.cc index f9d19ed73..4060c7780 100644 --- a/src/ufo/filters/obsfunctions/ObsErrorFactorTopoRad.cc +++ b/src/ufo/filters/obsfunctions/ObsErrorFactorTopoRad.cc @@ -122,10 +122,12 @@ void ObsErrorFactorTopoRad::compute(const ObsFilterData & in, } } else if (inst == "amsua" || inst == "atms") { // Set channel numbers - int ich544 = 0, ich549 = 0, ich890 = 0; + int ich238, ich314, ich503, ich528, ich536, ich544, ich549, ich890; if (inst == "amsua") { + ich238 = 1, ich314 = 2, ich503 = 3, ich528 = 4, ich536 = 5; ich544 = 6, ich549 = 7, ich890 = 15; } else if (inst == "atms") { + ich238 = 1, ich314 = 2, ich503 = 3, ich528 = 5, ich536 = 6; ich544 = 7, ich549 = 8, ich890 = 16; } diff --git a/src/ufo/filters/obsfunctions/SolarZenith.cc b/src/ufo/filters/obsfunctions/SolarZenith.cc index bd5cf0131..ddc161b95 100644 --- a/src/ufo/filters/obsfunctions/SolarZenith.cc +++ b/src/ufo/filters/obsfunctions/SolarZenith.cc @@ -98,9 +98,9 @@ void SolarZenith::compute(const ObsFilterData & in, ioda::ObsDataVector & util::DateTime dayStart = missingDateTime; util::DateTime dayEnd = missingDateTime; // Equation of time (more details below) - double eqnt = 0.0; + double eqnt; // Sine and cosine of declination - double sinDecl = 0.0, cosDecl = 0.0; + double sinDecl, cosDecl; for (size_t loc = 0; loc < nlocs; ++loc) { if (skipRejected && rejected[loc]) { diff --git a/src/ufo/filters/obsfunctions/TimeBinner.cc b/src/ufo/filters/obsfunctions/TimeBinner.cc deleted file mode 100644 index daaf94806..000000000 --- a/src/ufo/filters/obsfunctions/TimeBinner.cc +++ /dev/null @@ -1,334 +0,0 @@ -/* - * (C) Crown copyright 2025, Met Office - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include "ufo/filters/obsfunctions/TimeBinner.h" - -#include -#include -#include -#include -#include - -#include "ioda/ObsDataVector.h" -#include "oops/util/DateTime.h" -#include "oops/util/Duration.h" -#include "oops/util/missingValues.h" -#include "ufo/filters/ObsFilterData.h" - -namespace ufo { - -static ObsFunctionMaker makerObsFunctionTimeBinner_( - "TimeBinner"); - -ObsFunctionTimeBinner::ObsFunctionTimeBinner( - const eckit::LocalConfiguration &conf) { - oops::Log::trace() << "ObsFunctionTimeBinner constructor" << std::endl; - options_.validateAndDeserialize(conf); - - // Add the input timestamp variable to the required variables - invars_ += options_.inputTimestampVariable.value(); - - if (options_.binnedTimestampVariable.value() != boost::none) { - const boost::optional &binnedVar = - options_.binnedTimestampVariable.value(); - invars_ += *binnedVar; - } - - // Validate that the bin window does not overlap with adjacent bins - const util::Duration binIntervalUnit = - getBinIntervalUnit(options_.binIntervalUnit.value()); - const util::Duration binWindowLowerBound{ - options_.binWindowLowerBound.value()}; - const util::Duration binWindowUpperBound{ - options_.binWindowUpperBound.value()}; - - if (binWindowLowerBound > binWindowUpperBound) { - throw eckit::BadValue( - "The bin window lower bound must be less than or equal to the bin " - "window upper bound.", - Here()); - } - if ((binWindowUpperBound - binWindowLowerBound).toSeconds() >= - binIntervalUnit.toSeconds()) { - throw eckit::BadValue( - "Bin window (lower bound to upper bound) must be less than the bin " - "interval unit. " - "This ensures no overlap with adjacent bins.", - Here()); - } - // neither the bin window upper bound or lower bound can have a magnitude of - // greater than the bin interval unit, otherwise the flooring process to - // derive the bin label would be incorrect. - if (std::abs(binWindowLowerBound.toSeconds()) > binIntervalUnit.toSeconds() || - std::abs(binWindowUpperBound.toSeconds()) > binIntervalUnit.toSeconds()) { - throw eckit::BadValue( - "Bin window lower and upper bounds must not exceed the bin interval " - "unit in magnitude. " - "This ensures correct bin label derivation.", - Here()); - } -} - -ObsFunctionTimeBinner::~ObsFunctionTimeBinner() { - oops::Log::trace() << "ObsFunctionTimeBinner destructor" << std::endl; -} - -void ObsFunctionTimeBinner::compute(const ObsFilterData &in, - ioda::ObsDataVector &out) const { - oops::Log::trace() << "ObsFunctionTimeBinner compute start" << std::endl; - - const int missingBinValue = - util::missingValue(); // todo change to missingValue when this - // is allowed to be not just an IntObsFunction - - const size_t nlocs = in.nlocs(); - - // Retrieve input timestamps - std::vector timestamps(nlocs); - in.get(options_.inputTimestampVariable.value(), timestamps); - - // Parse bin interval and window - const util::Duration binIntervalUnit = - getBinIntervalUnit(options_.binIntervalUnit.value()); - const int64_t binIntervalUnitSecs = binIntervalUnit.toSeconds(); - const util::Duration binWindowLowerBound{ - options_.binWindowLowerBound.value()}; - const util::Duration binWindowUpperBound{ - options_.binWindowUpperBound.value()}; - const bool reverseOrder = options_.reverseChronologicalOrder.value(); - - // A fixed epoch is used for integer-second arithmetic - const util::DateTime epoch{"1970-01-01T00:00:00Z"}; - - // Bins are labelled with timestamps which are derived from a "floor" derived - // from the bin interval unit. Consider the timestamp 2018-04-17 01:20:02 UTC: - // if the bin interval unit is "hour" the floored time is 2018-04-17 01:00:00 - // UTC, if the bin interval unit is "minute" the floored time is 2018-04-17 - // 01:20:00 UTC, if the bin interval unit is "day" the floored time is - // 2018-04-17 00:00:00 UTC, etc. - // - // Which "floor"ed timestamp label is applied to a given timestamp depends on - // the bin window start and end. For example, with a bin interval unit of - // "hour" and a bin window start of -25 minutes and end of +25 minutes, the - // timestamp 2018-04-17 01:20:02 UTC would be labelled with the timestamp - // 2018-04-17 01:00:00 UTC (since it falls within the window of that bin), as - // would the timestamp 2018-04-17 00:50:00 UTC. All timestamps in that window - // would be assigned to the same bin number. - // - // Note that reverseOrder only affects the numbering of the bins, not the - // timestamps assigned to them. - // - // Bin windows do not have to cover the label timestamp. Bin window starts - // can be positive, implying that the window starts after the bin label - // timestamp. Bin window ends can also be negative, implying that the window - // ends before the bin label timestamp. However, the bin window (end - start) - // must not exceed the bin interval unit, to avoid overlap with adjacent bins - // and the start must be less than or equal to the end. - - // Iteratively find a valid reference timestamp that falls within a bin window - const util::DateTime firstBinLabelTimestamp = findFirstBinLabelTimestamp( - in.obsspace(), timestamps, binWindowLowerBound, binWindowUpperBound, - binIntervalUnit, epoch, reverseOrder); - - // Compute the bin numbers and optionally the binned timestamps - std::vector binLabelTimestamps( - nlocs, util::missingValue()); - if (firstBinLabelTimestamp == util::missingValue()) { - // No valid reference timestamp found, so all bin outputs are missing - std::fill(out[0].begin(), out[0].end(), missingBinValue); - } else { - // Assigns output bin index (respecting reverse ordering) and label - // timestamp. - const auto assignBin = [&](size_t iloc, int64_t bin, - const util::DateTime &binLabelTimestamp) { - out[0][iloc] = - reverseOrder ? static_cast(-bin) : static_cast(bin); - binLabelTimestamps[iloc] = binLabelTimestamp; - }; - // Returns true if the timestamp falls within the given bin label's window. - const auto inWindow = [&](const util::DateTime ×tamp, - const util::DateTime &binLabelTimestamp) { - return timestamp >= binLabelTimestamp + binWindowLowerBound && - timestamp <= binLabelTimestamp + binWindowUpperBound; - }; - - const int64_t firstBinLabelTimestampSecs = - (firstBinLabelTimestamp - epoch).toSeconds(); - for (size_t iloc = 0; iloc < nlocs; ++iloc) { - if (timestamps[iloc] == util::missingValue()) { - out[0][iloc] = missingBinValue; - continue; - } - // Find the candidate bin label timestamp for the current timestamp by - // flooring to the nearest bin interval unit. This uses modular arithmetic: - // timestampSecs - (timestampSecs % binIntervalUnitSecs) floors to the - // nearest multiple of binIntervalUnitSecs. Example: for a timestamp of - // 2018-04-17 01:20:02 UTC with hourly bins (binIntervalUnitSecs=3600): - // timestampSecs=1523928002, 1523928002 % 3600 = 1202, so the floor is - // 1523926800 which corresponds to 2018-04-17 01:00:00 UTC. - const int64_t timestampSecs = (timestamps[iloc] - epoch).toSeconds(); - const int64_t candidateBinLabelTimestampSecs = - timestampSecs - (timestampSecs % binIntervalUnitSecs); - const util::DateTime candidateBinLabelTimestamp = - epoch + util::Duration(candidateBinLabelTimestampSecs); - - // The candidate bin label should be exactly divisible by the bin interval - // unit, allowing the probable bin number to be deduced - assert((candidateBinLabelTimestampSecs % binIntervalUnitSecs) == 0); - int64_t candidateBinNumber = - (candidateBinLabelTimestampSecs - firstBinLabelTimestampSecs) / - binIntervalUnitSecs; - - if (inWindow(timestamps[iloc], candidateBinLabelTimestamp)) { - assignBin(iloc, candidateBinNumber, candidateBinLabelTimestamp); - continue; - } - // The timestamp doesn't fall within the candidate bin's window, but may - // still fall within the window of a neighboring bin, so check those - // before giving up and assigning missing. Note that only the - // immediately adjacent bins need to be checked, due to the validation - // that the bin window starts and ends cannot exceed the bin interval - // unit. Note also we don't check the previous bin if the candidate bin - // number is 0, since we don't allow negative bin numbers in the current - // implementation (see the documentation). - out[0][iloc] = missingBinValue; - const util::DateTime nextBinLabelTimestamp = - candidateBinLabelTimestamp + binIntervalUnit; - if (inWindow(timestamps[iloc], nextBinLabelTimestamp)) { - assignBin(iloc, candidateBinNumber + 1, nextBinLabelTimestamp); - } else if (candidateBinNumber > 0) { - const util::DateTime prevBinLabelTimestamp = - candidateBinLabelTimestamp - binIntervalUnit; - if (inWindow(timestamps[iloc], prevBinLabelTimestamp)) { - assignBin(iloc, candidateBinNumber - 1, prevBinLabelTimestamp); - } - } - } - } - // Save bin label timestamps if requested - if (options_.binnedTimestampVariable.value() != boost::none) { - const Variable &binnedVar = *options_.binnedTimestampVariable.value(); - in.obsspace().put_db(binnedVar.group(), binnedVar.variable(), - binLabelTimestamps); - } - - oops::Log::trace() << "ObsFunctionTimeBinner compute complete" << std::endl; -} - -const util::Duration ObsFunctionTimeBinner::getBinIntervalUnit( - const std::string &unit) const { - if (unit == "second" || unit == "seconds") { - return util::Duration("PT1S"); - } else if (unit == "minute" || unit == "minutes") { - return util::Duration("PT1M"); - } else if (unit == "hour" || unit == "hours") { - return util::Duration("PT1H"); - } else if (unit == "day" || unit == "days") { - return util::Duration("P1D"); - } else { - throw eckit::BadValue("Unsupported bin interval unit: " + unit, Here()); - } -} - -const ufo::Variables &ObsFunctionTimeBinner::requiredVariables() const { - return invars_; -} - -const util::DateTime ObsFunctionTimeBinner::findFirstBinLabelTimestamp( - const ioda::ObsSpace &obsSpace, - const std::vector ×tamps, - const util::Duration &binWindowLowerBound, - const util::Duration &binWindowUpperBound, - const util::Duration &binIntervalUnit, const util::DateTime &epoch, - bool reverseOrder) const { - // Find a valid reference timestamp (min or max, depending on order) that - // falls within a bin window. This determines the first bin label and allows - // all other bin numbers to be computed consistently. Iteratively try - // reference timestamps until one is found that fits, or return missing if - // none fit. - auto workingTimestamps = timestamps; - const int64_t binWindowLowerBoundSecs = binWindowLowerBound.toSeconds(); - const int64_t binIntervalUnitSecs = binIntervalUnit.toSeconds(); - - while (!workingTimestamps.empty()) { - // Select the earliest (chronological) or latest (reverse) timestamp. - const util::DateTime trialReferenceTimestamp = - reverseOrder - ? getGlobalMaxTimestamp(obsSpace, workingTimestamps, epoch) - : getGlobalMinTimestamp(obsSpace, workingTimestamps, epoch); - const int64_t trialReferenceTimestampSecs = - (trialReferenceTimestamp - epoch).toSeconds(); - - // Compute the candidate bin label for this reference timestamp. The label - // is derived by flooring to the bin interval unit, but first we adjust the - // timestamp to account for the window lower bound offset. This ensures that - // timestamps near (but after) a potential bin label are mapped to the - // correct label, not the previous one. For example: with a 1-hour interval - // and a relative window of [-30 min, +30 min), a reference at 19:40 - // belongs to the 20:00 bin window [19:30, 20:30), not the 19:00 window. - // The adjustment below handles this by shifting negative window lower - // bounds forward before flooring. - int64_t trialReferenceTimestampAdjustedForWindowSecs = - trialReferenceTimestampSecs; - if (binWindowLowerBoundSecs < 0) { - trialReferenceTimestampAdjustedForWindowSecs -= binWindowLowerBoundSecs; - } - - const int64_t candidateBinLabelTimestampSecs = - trialReferenceTimestampAdjustedForWindowSecs - - (trialReferenceTimestampAdjustedForWindowSecs % binIntervalUnitSecs); - const util::DateTime candidateBinLabelTimestamp = - epoch + util::Duration(candidateBinLabelTimestampSecs); - - // Check if the reference timestamp falls within this candidate bin's - // window. - const util::DateTime candidateBinWindowLowerBound = - candidateBinLabelTimestamp + binWindowLowerBound; - const util::DateTime candidateBinWindowUpperBound = - candidateBinLabelTimestamp + binWindowUpperBound; - - if (trialReferenceTimestamp >= candidateBinWindowLowerBound && - trialReferenceTimestamp <= candidateBinWindowUpperBound) { - // Found a valid first bin label. - return candidateBinLabelTimestamp; - } - - // This reference timestamp doesn't fit any bin window; remove it and - // iterate again using the next min/max timestamp. - workingTimestamps.erase( - std::remove(workingTimestamps.begin(), workingTimestamps.end(), - trialReferenceTimestamp), - workingTimestamps.end()); - } - return util::missingValue(); -} - -const util::DateTime ObsFunctionTimeBinner::getGlobalMinTimestamp( - const ioda::ObsSpace &obsSpace, - const std::vector ×tamps, - const util::DateTime &epoch) const { - const util::DateTime minTimestamp = *std::min_element( - timestamps.begin(), timestamps.end(), - [](const util::DateTime &a, const util::DateTime &b) { return a < b; }); - int64_t minTimestampSecs = (minTimestamp - epoch).toSeconds(); - obsSpace.comm().allReduceInPlace(minTimestampSecs, eckit::mpi::min()); - return epoch + util::Duration(minTimestampSecs); -} - -const util::DateTime ObsFunctionTimeBinner::getGlobalMaxTimestamp( - const ioda::ObsSpace &obsSpace, - const std::vector ×tamps, - const util::DateTime &epoch) const { - const util::DateTime maxTimestamp = *std::max_element( - timestamps.begin(), timestamps.end(), - [](const util::DateTime &a, const util::DateTime &b) { return a < b; }); - int64_t maxTimestampSecs = (maxTimestamp - epoch).toSeconds(); - obsSpace.comm().allReduceInPlace(maxTimestampSecs, eckit::mpi::max()); - return epoch + util::Duration(maxTimestampSecs); -} - -} // namespace ufo diff --git a/src/ufo/filters/obsfunctions/TimeBinner.h b/src/ufo/filters/obsfunctions/TimeBinner.h deleted file mode 100644 index ad97415ff..000000000 --- a/src/ufo/filters/obsfunctions/TimeBinner.h +++ /dev/null @@ -1,247 +0,0 @@ -/* - * (C) Crown copyright 2025, Met Office - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#ifndef UFO_FILTERS_OBSFUNCTIONS_TIMEBINNER_H_ -#define UFO_FILTERS_OBSFUNCTIONS_TIMEBINNER_H_ - -#include -#include - -#include "ioda/ObsSpace.h" -#include "oops/util/parameters/OptionalParameter.h" -#include "oops/util/parameters/Parameter.h" -#include "oops/util/parameters/Parameters.h" -#include "oops/util/parameters/RequiredParameter.h" -#include "ufo/filters/obsfunctions/ObsFunctionBase.h" -#include "ufo/filters/Variable.h" -#include "ufo/filters/Variables.h" -#include "ufo/utils/parameters/ParameterTraitsVariable.h" - -namespace ufo { - -/// \brief Parameters controlling IntObsFunction/TimeBinner. -/// -/// TimeBinner assigns each observation timestamp to a bin label timestamp and -/// returns a bin number as an integer ObsFunction output. -/// -/// Bin membership rule: -/// An observation timestamp t belongs to bin label T when -/// T + bin window lower bound <= t <= T + bin window upper bound -/// -/// Key constraints (validated in the implementation): -/// - bin window lower bound <= bin window upper bound -/// - (bin window upper bound - bin window lower bound) < bin interval unit -/// - |bin window lower bound| and |bin window upper bound| are each <= bin -/// interval unit -/// -/// If an observation falls outside all bin windows, the output bin number is -/// missing. -class TimeBinnerParameters : public oops::Parameters { - OOPS_CONCRETE_PARAMETERS(TimeBinnerParameters, Parameters) - - public: - /// \brief Bin interval granularity. - /// - /// Accepted values: second, seconds, minute, minutes, hour, hours, day, days. - oops::RequiredParameter binIntervalUnit{"bin interval unit", - this}; - - /// \brief Lower bound of the bin window relative to the bin label timestamp. - /// - /// Example values: -PT30M, PT0S, PT1S. - oops::RequiredParameter binWindowLowerBound{ - "bin window lower bound", this}; - - /// \brief Upper bound of the bin window relative to the bin label timestamp. - /// - /// Example values: PT0M, PT30M, PT1H. - oops::RequiredParameter binWindowUpperBound{ - "bin window upper bound", this}; - - /// \brief Input timestamp variable. - /// - /// Default: MetaData/dateTime. - oops::Parameter inputTimestampVariable{ - "input timestamp variable", Variable{"MetaData/dateTime"}, this}; - - /// \brief Optional output variable holding bin label timestamps. - /// - /// For observations assigned to bins, the label timestamp is written. - /// For observations outside all bin windows, missing DateTime is written. - oops::OptionalParameter binnedTimestampVariable{ - "binned timestamp variable", this}; - - /// \brief If true, bin numbering is reverse chronological. - /// - /// - false: bin 0 is the earliest valid bin label. - /// - true: bin 0 is the latest valid bin label. - oops::Parameter reverseChronologicalOrder{"reverse chronological order", - false, this}; -}; - -// ----------------------------------------------------------------------------- - -/// \brief Bins observation timestamps into discrete time intervals. -/// -/// This is an IntObsFunction registered as IntObsFunction/TimeBinner. -/// -/// The output integer is the bin number. Optionally, a second variable can be -/// populated with the associated bin label timestamp. -/// -/// Example A: window ending at the hour boundary -/// \code{.yaml} -/// - filter: Variable Assignment -/// assignments: -/// - name: MetaData/binNumbers -/// type: int -/// function: -/// name: IntObsFunction/TimeBinner -/// options: -/// bin interval unit: hour -/// bin window lower bound: -PT30M -/// bin window upper bound: PT0M -/// binned timestamp variable: MetaData/binTimestamps -/// \endcode -/// | input timestamp | bin number | binned timestamp | -/// |----------------------|------------|----------------------| -/// | 2018-04-16T23:50:00Z | 0 | 2018-04-17T00:00:00Z | -/// | 2018-04-17T00:15:00Z | missing | missing | -/// | 2018-04-17T00:35:00Z | 1 | 2018-04-17T01:00:00Z | -/// -/// Example B: window spanning the hour boundary -/// \code{.yaml} -/// - filter: Variable Assignment -/// assignments: -/// - name: MetaData/binNumbers -/// type: int -/// function: -/// name: IntObsFunction/TimeBinner -/// options: -/// bin interval unit: hour -/// bin window lower bound: -PT30M -/// bin window upper bound: PT29M59S -/// binned timestamp variable: MetaData/binTimestamps -/// \endcode -/// | input timestamp | bin number | binned timestamp | -/// |----------------------|------------|----------------------| -/// | 2018-04-17T00:15:00Z | 0 | 2018-04-17T00:00:00Z | -/// | 2018-04-17T00:35:00Z | 1 | 2018-04-17T01:00:00Z | -/// -/// Example C: reverse chronological order (same window as Example B) -/// \code{.yaml} -/// - filter: Variable Assignment -/// assignments: -/// - name: MetaData/binNumbers -/// type: int -/// function: -/// name: IntObsFunction/TimeBinner -/// options: -/// bin interval unit: hour -/// bin window lower bound: -PT30M -/// bin window upper bound: PT29M59S -/// binned timestamp variable: MetaData/binTimestamps -/// reverse chronological order: true -/// \endcode -/// | input timestamp | bin number | binned timestamp | -/// |----------------------|------------|----------------------| -/// | 2018-04-17T00:15:00Z | 1 | 2018-04-17T00:00:00Z | -/// | 2018-04-17T00:35:00Z | 0 | 2018-04-17T01:00:00Z | -class ObsFunctionTimeBinner : public ObsFunctionBase { - public: - /// \brief Construct TimeBinner from YAML/local configuration. - /// - /// Validates TimeBinner options and declares required input variables. - /// - /// \param conf Local configuration containing TimeBinner options. - explicit ObsFunctionTimeBinner(const eckit::LocalConfiguration &conf); - - /// \brief Destroy the TimeBinner instance. - ~ObsFunctionTimeBinner(); - - /// \brief Compute bin numbers for all observations. - /// - /// Writes integer bin numbers to \p out. If configured, also writes the - /// corresponding bin label timestamps to ObsSpace. Observations with missing - /// input timestamps receive missing (invalid) bin numbers. - /// - /// \param in Input data accessor providing observation values. - /// \param out Output integer vector storing bin numbers. - void compute(const ObsFilterData &in, - ioda::ObsDataVector &out) const override; - - /// \brief Return variables required by this ObsFunction. - /// - /// Includes the input timestamp variable and optionally the configured output - /// binned timestamp variable. - /// - /// \return Set of variables required by TimeBinner. - const ufo::Variables &requiredVariables() const override; - - private: - TimeBinnerParameters options_; - ufo::Variables invars_; - - /// \brief Find the first valid bin label timestamp used as bin-number origin. - /// - /// Iteratively selects a trial reference timestamp (min or max depending on - /// \p reverseOrder), derives a candidate bin label, and returns the first - /// candidate whose window contains that trial timestamp. If no timestamps fit - /// into any bin window, returns missing DateTime. - /// - /// \param obsSpace ObsSpace used for MPI-aware min/max operations. - /// \param timestamps Candidate timestamps (must not be empty). - /// \param binWindowLowerBound Window lower bound offset from bin label. - /// \param binWindowUpperBound Window upper bound offset from bin label. - /// \param binIntervalUnit Bin interval unit duration (hour, minute, second, - /// day). - /// \param epoch Epoch used for integer-second arithmetic. - /// \param reverseOrder If true, select global max first; otherwise min first. - /// \return First valid bin label timestamp, or missing DateTime if none - /// found. - const util::DateTime findFirstBinLabelTimestamp( - const ioda::ObsSpace &obsSpace, - const std::vector ×tamps, - const util::Duration &binWindowLowerBound, - const util::Duration &binWindowUpperBound, - const util::Duration &binIntervalUnit, const util::DateTime &epoch, - bool reverseOrder) const; - - /// \brief Return global minimum timestamp across MPI ranks. - /// - /// \param obsSpace ObsSpace providing communicator. - /// \param timestamps Local timestamps (must not be empty). - /// \param epoch Epoch used for integer-second arithmetic. - /// \return Global minimum timestamp. - const util::DateTime getGlobalMinTimestamp( - const ioda::ObsSpace &obsSpace, - const std::vector ×tamps, - const util::DateTime &epoch) const; - - /// \brief Return global maximum timestamp across MPI ranks. - /// - /// \param obsSpace ObsSpace providing communicator. - /// \param timestamps Local timestamps (must not be empty). - /// \param epoch Epoch used for integer-second arithmetic. - /// \return Global maximum timestamp. - const util::DateTime getGlobalMaxTimestamp( - const ioda::ObsSpace &obsSpace, - const std::vector ×tamps, - const util::DateTime &epoch) const; - - /// \brief Convert bin interval unit string to util::Duration. - /// - /// Supported units are second(s), minute(s), hour(s), and day(s). - /// - /// \param unit Unit string from configuration. - /// \return Duration corresponding to \p unit. - /// \throw eckit::BadValue if \p unit is unsupported. - const util::Duration getBinIntervalUnit(const std::string &unit) const; -}; - -} // namespace ufo - -#endif // UFO_FILTERS_OBSFUNCTIONS_TIMEBINNER_H_ diff --git a/src/ufo/filters/refractivityonedvarcheck/RefractivityOneDVarCheck.cc b/src/ufo/filters/refractivityonedvarcheck/RefractivityOneDVarCheck.cc index 73584558a..f5bd27dc7 100644 --- a/src/ufo/filters/refractivityonedvarcheck/RefractivityOneDVarCheck.cc +++ b/src/ufo/filters/refractivityonedvarcheck/RefractivityOneDVarCheck.cc @@ -29,8 +29,8 @@ namespace ufo { RefractivityOneDVarCheck::RefractivityOneDVarCheck( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr + std::shared_ptr > flags, + std::shared_ptr > obserr ) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { @@ -94,8 +94,8 @@ void RefractivityOneDVarCheck::applyFilter( } // Save qc flags to database for retrieval in fortran - needed for channel selection - flags_.save("FortranQC"); // temporary measure as per ROobserror qc - obserr_.save("FortranERR"); // Pass latest errors to 1DVar + flags_->save("FortranQC"); // temporary measure as per ROobserror qc + obserr_->save("FortranERR"); // Pass latest errors to 1DVar // Pass it all to fortran ufo_refractivityonedvarcheck_apply_f90( @@ -105,7 +105,7 @@ void RefractivityOneDVarCheck::applyFilter( apply_char[0]); // Read qc flags from database - flags_.read("FortranQC"); // temporary measure as per ROobserror qc + flags_->read("FortranQC"); // temporary measure as per ROobserror qc oops::Log::trace() << "RefractivityOneDVarCheck Filter complete" << std::endl; } diff --git a/src/ufo/filters/refractivityonedvarcheck/RefractivityOneDVarCheck.h b/src/ufo/filters/refractivityonedvarcheck/RefractivityOneDVarCheck.h index ccdc622f8..6d1a5b555 100644 --- a/src/ufo/filters/refractivityonedvarcheck/RefractivityOneDVarCheck.h +++ b/src/ufo/filters/refractivityonedvarcheck/RefractivityOneDVarCheck.h @@ -58,8 +58,8 @@ class RefractivityOneDVarCheck : public FilterBase, RefractivityOneDVarCheck( ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~RefractivityOneDVarCheck(); private: diff --git a/src/ufo/filters/rttovonedvarcheck/RTTOVOneDVarCheck.cc b/src/ufo/filters/rttovonedvarcheck/RTTOVOneDVarCheck.cc index 1c6719d82..29dad9cf2 100644 --- a/src/ufo/filters/rttovonedvarcheck/RTTOVOneDVarCheck.cc +++ b/src/ufo/filters/rttovonedvarcheck/RTTOVOneDVarCheck.cc @@ -27,8 +27,8 @@ namespace ufo { // ----------------------------------------------------------------------------- RTTOVOneDVarCheck::RTTOVOneDVarCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), channels_(), retrieved_vars_(), hoxdiags_retrieved_vars_(), parameters_(parameters) { @@ -117,7 +117,7 @@ void RTTOVOneDVarCheck::applyFilter(const std::vector & apply, } // Save qc flags to database for retrieval in fortran - needed for channel selection - flags_.save("FortranQC"); // temporary measure as per ROobserror qc + flags_->save("FortranQC"); // temporary measure as per ROobserror qc // Read in ObsBias for all channels in the database and save to db for access in Fortran // there is currently no mechanism for passing ioda::ObsDataVectors to Fortran. @@ -138,7 +138,7 @@ void RTTOVOneDVarCheck::applyFilter(const std::vector & apply, apply_char.size(), apply_char[0]); // Read qc flags from database - flags_.read("FortranQC"); // temporary measure as per ROobserror qc + flags_->read("FortranQC"); // temporary measure as per ROobserror qc oops::Log::trace() << "RTTOVOneDVarCheck applyFilter complete" << std::endl; } diff --git a/src/ufo/filters/rttovonedvarcheck/RTTOVOneDVarCheck.h b/src/ufo/filters/rttovonedvarcheck/RTTOVOneDVarCheck.h index db2df0ae6..8b036526b 100644 --- a/src/ufo/filters/rttovonedvarcheck/RTTOVOneDVarCheck.h +++ b/src/ufo/filters/rttovonedvarcheck/RTTOVOneDVarCheck.h @@ -54,8 +54,8 @@ class RTTOVOneDVarCheck : public FilterBase, static const std::string classname() {return "ufo::RTTOVOneDVarCheck";} RTTOVOneDVarCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~RTTOVOneDVarCheck() override; private: diff --git a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_ob_mod.f90 b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_ob_mod.f90 index e1d60c21f..0c47fa351 100644 --- a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_ob_mod.f90 +++ b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_ob_mod.f90 @@ -243,10 +243,8 @@ subroutine ufo_rttovonedvarcheck_PrintOb(self) write(*,"(10F8.2)") self % background_T write(*,"(A)") "Emissivity: " write(*,"(10F8.2)") self % emiss(:) -if (allocated(self % pcemiss)) then - write(*,"(A)") "Emissivity PC: " - write(*,"(10F18.8)") self % pcemiss(:) -endif +write(*,"(A)") "Emissivity PC: " +write(*,"(10F18.8)") self % pcemiss(:) end subroutine diff --git a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_setup_mod.f90 b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_setup_mod.f90 index 7b23fe350..138af5d8b 100644 --- a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_setup_mod.f90 +++ b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_setup_mod.f90 @@ -403,11 +403,9 @@ subroutine ufo_rttovonedvarcheck_print(self) write(*,*) "EmissLandDefault = ",self % EmissLandDefault write(*,*) "EmissSeaIceDefault = ",self % EmissSeaIceDefault write(*,*) "mwEmissRetrieval = ",self % mwEmissRetrieval -if (self % mwEmissRetrieval) then - write(*,*) "NumEmissElements = ",self % NumEmissElements - write(*,*) "EmissToChannelMap = ",self % EmissToChannelMap - write(*,*) "ChannelToEmissMap = ",self % ChannelToEmissMap -endif +write(*,*) "NumEmissElements = ",self % NumEmissElements +write(*,*) "EmissToChannelMap = ",self % EmissToChannelMap +write(*,*) "ChannelToEmissMap = ",self % ChannelToEmissMap write(*,*) "Use PC for Emissivity = ", self % pcemiss write(*,*) "EmisEigVecPath = ",self % EmisEigVecPath write(*,*) "EmisAtlas = ",self % EmisAtlas diff --git a/src/ufo/fov/calc_fov_conical.f90 b/src/ufo/fov/calc_fov_conical.f90 index 9ca3dbdf5..312f984ac 100644 --- a/src/ufo/fov/calc_fov_conical.f90 +++ b/src/ufo/fov/calc_fov_conical.f90 @@ -1032,11 +1032,7 @@ subroutine inside_fov_conical(instr,ichan,satellite_azimuth,lat,lon, & distance_east = r2*cos(lat*deg2rad)*dellon ! Angle to the test point - if (distance_north == 0 .and. distance_east == 0) then - bearing_to_test = 0 - else - bearing_to_test = mod(atan2(distance_north,distance_east),2*pi ) - endif + bearing_to_test = mod(atan2(distance_north,distance_east),two*pi) bearing_to_test_deg = bearing_to_test*rad2deg ! convert to degrees ! This is the arc distance to the test point diff --git a/src/ufo/fov/calc_fov_crosstrk.f90 b/src/ufo/fov/calc_fov_crosstrk.f90 index 7a2c49072..39f4e86a0 100644 --- a/src/ufo/fov/calc_fov_crosstrk.f90 +++ b/src/ufo/fov/calc_fov_crosstrk.f90 @@ -1360,11 +1360,7 @@ subroutine inside_fov_crosstrk(instr, ifov, satellite_azimuth, & distance_east = r2*cos(lat*deg2rad)*dellon ! angle to the test point - if (distance_north == 0 .and. distance_east == 0) then - bearing_to_test = 0 - else - bearing_to_test = mod(atan2(distance_north,distance_east),2*pi ) - endif + bearing_to_test = mod(atan2(distance_north,distance_east),2*pi ) bearing_to_test_deg = bearing_to_test*rad2deg ! convert to degrees ! this is the arc distance to the test point diff --git a/src/ufo/obslocalization/ObsHorLocGC99.h b/src/ufo/obslocalization/ObsHorLocGC99.h index d6a3f92b1..76b33e51b 100644 --- a/src/ufo/obslocalization/ObsHorLocGC99.h +++ b/src/ufo/obslocalization/ObsHorLocGC99.h @@ -31,8 +31,6 @@ class ObsHorLocGC99: public ufo::ObsHorLocalization { public: ObsHorLocGC99(const eckit::Configuration &, ioda::ObsSpace &); - double computeLocalization(const eckit::geometry::Point3 &, - const eckit::geometry::Point3 &) const override; protected: /// Compute GC99 localization using the set of \p localobs, the same lengthscale /// used to search for those obs, and save localization values in \p locvector. @@ -54,20 +52,6 @@ ObsHorLocGC99::ObsHorLocGC99(const eckit::Configuration & conf, // ----------------------------------------------------------------------------- -template -double ObsHorLocGC99::computeLocalization( - const eckit::geometry::Point3 & p1, - const eckit::geometry::Point3 & p2) const { - eckit::geometry::Point2 p1_2(p1[0], p1[1]); - eckit::geometry::Point2 p2_2(p2[0], p2[1]); - double distance = atlas::util::Earth::distance(p1_2, p2_2); - double lengthscale = ObsHorLocalization::lengthscale(); - double loc = oops::gc99(distance / lengthscale); - return (distance >= lengthscale) ? 0.0 : loc; -} - -// ----------------------------------------------------------------------------- - template void ObsHorLocGC99::localizeLocalObs(const ITERATOR & i, ioda::ObsVector & locvector, diff --git a/src/ufo/obslocalization/ObsHorLocSOAR.h b/src/ufo/obslocalization/ObsHorLocSOAR.h index 4404d842d..101e769d8 100644 --- a/src/ufo/obslocalization/ObsHorLocSOAR.h +++ b/src/ufo/obslocalization/ObsHorLocSOAR.h @@ -31,8 +31,6 @@ class ObsHorLocSOAR: public ufo::ObsHorLocalization { public: ObsHorLocSOAR(const eckit::Configuration &, ioda::ObsSpace &); - double computeLocalization(const eckit::geometry::Point3 &, - const eckit::geometry::Point3 &) const override; protected: /// Compute SOAR localization using the set of \p localobs and save localization /// values in \p locvector. @@ -58,20 +56,6 @@ ObsHorLocSOAR::ObsHorLocSOAR(const eckit::Configuration & config, // ----------------------------------------------------------------------------- -template -double ObsHorLocSOAR::computeLocalization( - const eckit::geometry::Point3 & p1, - const eckit::geometry::Point3 & p2) const { - eckit::geometry::Point2 p1_2(p1[0], p1[1]); - eckit::geometry::Point2 p2_2(p2[0], p2[1]); - double distance = atlas::util::Earth::distance(p1_2, p2_2); - double lengthscale = ObsHorLocalization::lengthscale(); - double loc = oops::soar(distance * options_.SOARexpDecayH); - return (distance >= lengthscale) ? 0.0 : loc; -} - -// ----------------------------------------------------------------------------- - template void ObsHorLocSOAR::localizeLocalObs(const ITERATOR & i, ioda::ObsVector & locvector, diff --git a/src/ufo/obslocalization/ObsHorLocalization.h b/src/ufo/obslocalization/ObsHorLocalization.h index 51484f090..03798a57a 100644 --- a/src/ufo/obslocalization/ObsHorLocalization.h +++ b/src/ufo/obslocalization/ObsHorLocalization.h @@ -32,10 +32,6 @@ #include "ufo/obslocalization/ObsHorLocParameters.h" #include "ufo/obslocalization/ObsLocalizationBase.h" -namespace eckit::geometry { - class Point3; -} - namespace ufo { /// Horizontal Box car observation space localization @@ -50,14 +46,6 @@ class ObsHorLocalization: public ObsLocalizationBase { void computeLocalization(const ITERATOR &, ioda::ObsVector & locvector) const override; - /// Compute localization values between two points (regardless of if those are MODEL/OBS - /// or OBS/OBS locations) and return the localization value. The lengthscale from - /// ObsHorLocParameters is used. - /// Return values should be between 0.0 and 1.0, inclusive, with 0 indicating that the points - /// are outside of localization. - double computeLocalization(const eckit::geometry::Point3 &, - const eckit::geometry::Point3 &) const override; - protected: struct LocalObs { /// The list of indexes for ObsVector pointing to the valid local obs. @@ -185,18 +173,6 @@ void ObsHorLocalization::computeLocalization(const ITERATOR & i, // ----------------------------------------------------------------------------- -template -double ObsHorLocalization::computeLocalization( - const eckit::geometry::Point3 & p1, - const eckit::geometry::Point3 & p2) const { - eckit::geometry::Point2 p1_2(p1[0], p1[1]); - eckit::geometry::Point2 p2_2(p2[0], p2[1]); - double distance = atlas::util::Earth::distance(p1_2, p2_2); - return (distance >= options_.lengthscale) ? 0.0 : 1.0; -} - -// ----------------------------------------------------------------------------- - template void ObsHorLocalization::localizeLocalObs(const ITERATOR & i, ioda::ObsVector & locvector, diff --git a/src/ufo/obslocalization/ObsLocalization.h b/src/ufo/obslocalization/ObsLocalization.h index b03048b2f..58f7d03e7 100644 --- a/src/ufo/obslocalization/ObsLocalization.h +++ b/src/ufo/obslocalization/ObsLocalization.h @@ -30,8 +30,6 @@ class ObsLocalization : public util::Printable { ~ObsLocalization(); void computeLocalization(const ITERATOR &, ioda::ObsVector &) const; - double computeLocalization(const eckit::geometry::Point3 &, - const eckit::geometry::Point3 &) const; private: void print(std::ostream &) const override; @@ -63,14 +61,6 @@ void ObsLocalization::computeLocalization(const ITERATOR & iter, // ----------------------------------------------------------------------------- -template -double ObsLocalization::computeLocalization(const eckit::geometry::Point3 & p1, - const eckit::geometry::Point3 & p2) const { - return obsloc_->computeLocalization(p1, p2); -} - -// ----------------------------------------------------------------------------- - template void ObsLocalization::print(std::ostream & os) const { os << *obsloc_; diff --git a/src/ufo/obslocalization/ObsLocalizationBase.h b/src/ufo/obslocalization/ObsLocalizationBase.h index afde57d41..130f6b61e 100644 --- a/src/ufo/obslocalization/ObsLocalizationBase.h +++ b/src/ufo/obslocalization/ObsLocalizationBase.h @@ -13,7 +13,6 @@ #include #include "eckit/config/Configuration.h" -#include "eckit/geometry/Point3.h" #include "ioda/ObsSpace.h" #include "ioda/ObsVector.h" #include "oops/util/Logger.h" @@ -33,17 +32,6 @@ class ObsLocalizationBase : public util::Printable, /// localization values between observations and \p point in model-space. /// Set \p locfactor to missing value for observations that are not local. virtual void computeLocalization(const ITERATOR & point, ioda::ObsVector & locfactor) const = 0; - - /// compute localization between an observation and a point in model-space, or - /// between two observations. Return values should be between 0.0 and 1.0, inclusive, - /// with 0 indicating that the points are outside of localization. - virtual double computeLocalization(const eckit::geometry::Point3 &, - const eckit::geometry::Point3 &) const { - // NOTE, this really should be a pure virtual method, but since not all - // ObsLocalizationBase derived classes will be implementing this method yet, - // we are providing a default implementation. - ABORT("computeLocalization(Point3,Point3) not implemented for this ObsLocalization class"); - return 0.0;} }; // ============================================================================= diff --git a/src/ufo/operators/crtm/ObsRadianceCRTM.cc b/src/ufo/operators/crtm/ObsRadianceCRTM.cc index 3026b0b3d..4824012b1 100644 --- a/src/ufo/operators/crtm/ObsRadianceCRTM.cc +++ b/src/ufo/operators/crtm/ObsRadianceCRTM.cc @@ -503,7 +503,7 @@ void ObsRadianceCRTM::simulateObs(const GeoVaLs & gom, ioda::ObsVector & ovec, ufo_radiancecrtm_simobs_f90(keyOperRadianceCRTM_, gom.toFortran(), odb_, ovec.nvars(), ovec.nlocs(), ovec.toFortran(), dvec.toFortran(), - qc_flags); + reinterpret_cast(&qc_flags)); oops::Log::trace() << "ObsRadianceCRTM::simulateObs done" << std::endl; } diff --git a/src/ufo/operators/crtm/ObsRadianceCRTM.interface.h b/src/ufo/operators/crtm/ObsRadianceCRTM.interface.h index 627cd5179..9f9c2e323 100644 --- a/src/ufo/operators/crtm/ObsRadianceCRTM.interface.h +++ b/src/ufo/operators/crtm/ObsRadianceCRTM.interface.h @@ -40,8 +40,8 @@ extern "C" { oops::Variables &, const eckit::mpi::Comm &); void ufo_radiancecrtm_delete_f90(F90hop &); void ufo_radiancecrtm_simobs_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, - const size_t &, const size_t &, double &, const F90goms &, - const ioda::ObsDataVector &); + const size_t &, const size_t &, double &, const F90goms &, + const void*); // ----------------------------------------------------------------------------- } // extern C diff --git a/src/ufo/operators/crtm/ObsRadianceCRTMTLAD.cc b/src/ufo/operators/crtm/ObsRadianceCRTMTLAD.cc index 2402aa7ed..6fc570e37 100644 --- a/src/ufo/operators/crtm/ObsRadianceCRTMTLAD.cc +++ b/src/ufo/operators/crtm/ObsRadianceCRTMTLAD.cc @@ -61,7 +61,7 @@ ObsRadianceCRTMTLAD::~ObsRadianceCRTMTLAD() { void ObsRadianceCRTMTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics & ydiags, const QCFlags_t & qc_flags) { ufo_radiancecrtm_tlad_settraj_f90(keyOperRadianceCRTM_, geovals.toFortran(), obsspace(), - ydiags.toFortran(), qc_flags); + ydiags.toFortran(), reinterpret_cast(&qc_flags)); oops::Log::trace() << "ObsRadianceCRTMTLAD::setTrajectory done" << std::endl; } @@ -69,7 +69,7 @@ void ObsRadianceCRTMTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics void ObsRadianceCRTMTLAD::simulateObsTL(const GeoVaLs & geovals, ioda::ObsVector & ovec) const { ufo_radiancecrtm_simobs_tl_f90(keyOperRadianceCRTM_, geovals.toFortran(), obsspace(), - ovec.nvars(), ovec.nlocs(), ovec.toFortran()); + ovec.nvars(), ovec.nlocs(), ovec.toFortran()); oops::Log::trace() << "ObsRadianceCRTMTLAD::simulateObsTL done" << std::endl; } diff --git a/src/ufo/operators/crtm/ObsRadianceCRTMTLAD.interface.h b/src/ufo/operators/crtm/ObsRadianceCRTMTLAD.interface.h index 5aab917cb..705c2ded9 100644 --- a/src/ufo/operators/crtm/ObsRadianceCRTMTLAD.interface.h +++ b/src/ufo/operators/crtm/ObsRadianceCRTMTLAD.interface.h @@ -41,11 +41,11 @@ extern "C" { oops::Variables &, const eckit::mpi::Comm &); void ufo_radiancecrtm_tlad_delete_f90(F90hop &); void ufo_radiancecrtm_tlad_settraj_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, - const F90goms &, const ioda::ObsDataVector &); + const F90goms &, const void*); void ufo_radiancecrtm_simobs_tl_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, - const int &, const int &, double &); + const int &, const int &, double &); void ufo_radiancecrtm_simobs_ad_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, - const int &, const int &, const double &); + const int &, const int &, const double &); // ----------------------------------------------------------------------------- } // extern C diff --git a/src/ufo/operators/gnssro/BendMetOffice/ufo_gnssro_bendmetoffice_tlad_mod.F90 b/src/ufo/operators/gnssro/BendMetOffice/ufo_gnssro_bendmetoffice_tlad_mod.F90 index 763f17447..f8c3e733b 100644 --- a/src/ufo/operators/gnssro/BendMetOffice/ufo_gnssro_bendmetoffice_tlad_mod.F90 +++ b/src/ufo/operators/gnssro/BendMetOffice/ufo_gnssro_bendmetoffice_tlad_mod.F90 @@ -160,11 +160,9 @@ subroutine ufo_gnssro_bendmetoffice_tlad_settraj(self, geovals, obss) ! Check that the number of vertical levels in the observations is consistent ! with what we were given in setup - if (self % nlevels > 1) then - if (self % nlevels /= size(self % chanList)) then - write(err_msg,'(2A,4I8)') myname_, ' error: channel list must match nlevels', self % nlevels, size(self%chanList) - call fckit_exception%throw(err_msg) - end if + if (self % nlevels > 1 .AND. self % nlevels /= size(self % chanList)) then + write(err_msg,'(2A,4I8)') myname_, ' error: channel list must match nlevels', self % nlevels, size(self%chanList) + call fckit_exception%throw(err_msg) end if ! Get the meta-data from the observations diff --git a/src/ufo/operators/gnssro/QC/BackgroundCheckRONBAM.cc b/src/ufo/operators/gnssro/QC/BackgroundCheckRONBAM.cc index f3ae313f9..a49d23660 100644 --- a/src/ufo/operators/gnssro/QC/BackgroundCheckRONBAM.cc +++ b/src/ufo/operators/gnssro/QC/BackgroundCheckRONBAM.cc @@ -30,8 +30,8 @@ namespace ufo { BackgroundCheckRONBAM::BackgroundCheckRONBAM(ioda::ObsSpace & obsdb, const eckit::Configuration & config, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, config, flags, obserr) { oops::Log::trace() << "BackgroundCheckRONBAM contructor start " @@ -75,7 +75,7 @@ void BackgroundCheckRONBAM::applyFilter(const std::vector & apply, data_.get(varhofx.variable(jv), hofx); for (size_t jobs = 0; jobs < obsdb_.nlocs(); ++jobs) { - if (apply[jobs] && flags_[iv][jobs] == 0) { + if (apply[jobs] && (*flags_)[iv][jobs] == 0) { ASSERT(obs[jv][jobs] != missing); ASSERT(hofx[jobs] != missing); ASSERT(impactparameter[0][jobs] != missing); diff --git a/src/ufo/operators/gnssro/QC/BackgroundCheckRONBAM.h b/src/ufo/operators/gnssro/QC/BackgroundCheckRONBAM.h index b1d6e31ea..c2418880d 100644 --- a/src/ufo/operators/gnssro/QC/BackgroundCheckRONBAM.h +++ b/src/ufo/operators/gnssro/QC/BackgroundCheckRONBAM.h @@ -36,8 +36,8 @@ class BackgroundCheckRONBAM : public FilterBase, static const std::string classname() {return "ufo::BackgroundCheckRONBAM";} BackgroundCheckRONBAM(ioda::ObsSpace &, const eckit::Configuration &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~BackgroundCheckRONBAM(); private: diff --git a/src/ufo/operators/gnssro/QC/ROobserror.cc b/src/ufo/operators/gnssro/QC/ROobserror.cc index 1f58737e6..8e3f74add 100644 --- a/src/ufo/operators/gnssro/QC/ROobserror.cc +++ b/src/ufo/operators/gnssro/QC/ROobserror.cc @@ -21,9 +21,9 @@ namespace ufo { ROobserror::ROobserror(ioda::ObsSpace & obsdb, const eckit::Configuration & config, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) - : FilterBase(obsdb, config, flags, obserr) + std::shared_ptr > qc, + std::shared_ptr > oberr) + : FilterBase(obsdb, config, qc, oberr) { oops::Log::trace() << "ROobserror contructor start" << std::endl; // observation errors are always set up for the variables that are @@ -54,11 +54,11 @@ void ROobserror::applyFilter(const std::vector & apply, std::vector> & flagged) const { oops::Log::trace() << "ROobserror::applyFilter using priorFilter start" << std::endl; // Call the fortran routines to do the processing - flags_.save("FortranQC"); // should pass values to fortran properly - obserr_.save("FortranERR"); // should pass values to fortran properly + flags_->save("FortranQC"); // should pass values to fortran properly + obserr_->save("FortranERR"); // should pass values to fortran properly ufo_roobserror_prior_f90(key_); - flags_.read("FortranQC"); // should get values from fortran properly - obserr_.read("FortranERR"); // should get values from fortran properly + flags_->read("FortranQC"); // should get values from fortran properly + obserr_->read("FortranERR"); // should get values from fortran properly oops::Log::trace() << "ROobserror::applyFilter using priorFilter done" << std::endl; } diff --git a/src/ufo/operators/gnssro/QC/ROobserror.h b/src/ufo/operators/gnssro/QC/ROobserror.h index 777f77c29..b4f786cc7 100644 --- a/src/ufo/operators/gnssro/QC/ROobserror.h +++ b/src/ufo/operators/gnssro/QC/ROobserror.h @@ -38,8 +38,8 @@ class ROobserror : public FilterBase, static const std::string classname() {return "ufo::ROobserror";} ROobserror(ioda::ObsSpace &, const eckit::Configuration &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ROobserror(); private: diff --git a/src/ufo/operators/marine/adt/ObsADT.cc b/src/ufo/operators/marine/adt/ObsADT.cc index 78019bc3c..7c5921a72 100644 --- a/src/ufo/operators/marine/adt/ObsADT.cc +++ b/src/ufo/operators/marine/adt/ObsADT.cc @@ -61,38 +61,63 @@ void ObsADT::simulateObs(const GeoVaLs & geovals, ioda::ObsVector & ovec, oops::Log::trace() << "ObsADT::simulateObs start" << std::endl; const double missing = util::missingValue(); - - // get geovals - std::vector vec(ovec.nlocs()); - geovals.getAtLevel(vec, oops::Variable{"sea_surface_height_above_geoid"}, 0); - - // get obs + const size_t nlocs = ovec.nlocs(); + + // ----------------------- + // 1. GeoVaLs + // ----------------------- + std::vector vec(nlocs); + geovals.getAtLevel(vec, + oops::Variable{"sea_surface_height_above_geoid"}, 0); + + // ----------------------- + // 2. Observations + // ----------------------- std::vector obs; - odb_.get_db("ObsValue", ovec.varnames().variables()[operatorVarIndex_], obs); + odb_.get_db("ObsValue", + ovec.varnames().variables()[operatorVarIndex_], + obs); + + // ----------------------- + // 3. Compute global offset (QC-filtered) + // ----------------------- + double offset = 0.0; - // calculate global offsets - double offset = 0; auto accumVal = odb_.distribution()->createAccumulator(); auto accumCnt = odb_.distribution()->createAccumulator(); - for (size_t jloc = 0; jloc < ovec.nlocs(); ++jloc) { - // TODO(travis) also remove obs that have failed QC before this point? - // Not doing it now to keep answers from changing with this PR. - if (obs[jloc] != missing && vec[jloc] != missing) { + + for (size_t jloc = 0; jloc < nlocs; ++jloc) { + // ✅ CORRECT QC ACCESS FOR ObsDataVector + const int qc = qc_flags[operatorVarIndex_][jloc]; + + if (qc == 0 && + obs[jloc] != missing && + vec[jloc] != missing && + std::isfinite(obs[jloc]) && + std::isfinite(vec[jloc])) { + accumVal->addTerm(jloc, vec[jloc] - obs[jloc]); accumCnt->addTerm(jloc, 1); } } - auto count = accumCnt->computeResult(); + + const int count = accumCnt->computeResult(); + if (count > 0) { offset = accumVal->computeResult() / count; } - // oops::Log::debug() << "ObsADT::simulateObs offset: " << offset << std::endl; - // subtract offset from geoval - for (size_t jloc = 0; jloc < ovec.nlocs(); ++jloc) { + // ----------------------- + // 4. Apply offset (no QC filtering here!) + // ----------------------- + for (size_t jloc = 0; jloc < nlocs; ++jloc) { const size_t idx = jloc * ovec.nvars() + operatorVarIndex_; + ovec[idx] = vec[jloc]; - if (ovec[idx] != missing) ovec[idx] -= offset; + + if (ovec[idx] != missing && std::isfinite(ovec[idx])) { + ovec[idx] -= offset; + } } oops::Log::trace() << "ObsADT::simulateObs done" << std::endl; diff --git a/src/ufo/operators/rttov/CPP/v14/rttovcpp_interface.cc b/src/ufo/operators/rttov/CPP/v14/rttovcpp_interface.cc index 898ad2b17..975c13056 100644 --- a/src/ufo/operators/rttov/CPP/v14/rttovcpp_interface.cc +++ b/src/ufo/operators/rttov/CPP/v14/rttovcpp_interface.cc @@ -220,7 +220,7 @@ void rttovcpp_interface(const GeoVaLs & geovals, const ioda::ObsSpace & odb_, //--------------------------------------------------------------------------------- util::DateTime time1; int year, month, day, hour, minute, second; - int surftype = 0; + int surftype; int isurf = 0; for (std::size_t i = 0; i < nprofiles; i++) { @@ -369,6 +369,12 @@ void rttovcpp_interface(const GeoVaLs & geovals, const ioda::ObsSpace & odb_, // get ydiag object ufo::GeoVaLs & ydiag = d.geovals(); + // get variables in ydiag; just for debug + const oops::Variables & vars = ydiag.getVars(); + for (const oops::Variable var : vars) { + oops::Log::trace() << "ydiag name: " << var << std::endl; + } + size_t iSurface = 0; // support only one surface type for now. for (const std::string &varname : ydiag_varnames) { // check varname dimension diff --git a/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOV.cc b/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOV.cc index 11384477f..a07328157 100644 --- a/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOV.cc +++ b/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOV.cc @@ -74,8 +74,8 @@ ObsRadianceRTTOV::~ObsRadianceRTTOV() { void ObsRadianceRTTOV::simulateObs(const GeoVaLs & gom, ioda::ObsVector & ovec, ObsDiagnostics & dvec, const QCFlags_t& qc_flags) const { ufo_radiancerttov_simobs_f90(keyOperRadianceRTTOV_, gom.toFortran(), odb_, - ovec.nvars(), ovec.nlocs(), ovec.toFortran(), - dvec.toFortran(), qc_flags); + ovec.nvars(), ovec.nlocs(), ovec.toFortran(), + dvec.toFortran(), reinterpret_cast(&qc_flags)); oops::Log::trace() << "ObsRadianceRTTOV::simulateObs done." << std::endl; } diff --git a/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOV.interface.h b/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOV.interface.h index 8bebe8a88..fa1834276 100644 --- a/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOV.interface.h +++ b/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOV.interface.h @@ -26,12 +26,12 @@ extern "C" { // Radiance observation operator // ----------------------------------------------------------------------------- void ufo_radiancerttov_setup_f90(F90hop &, const eckit::Configuration &, - const int &, const int &, oops::Variables &, - const int &, const int &); + const int &, const int &, oops::Variables &, + const int &, const int &); void ufo_radiancerttov_delete_f90(F90hop &); void ufo_radiancerttov_simobs_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, - const int &, const int &, double &, const F90goms &, - const ioda::ObsDataVector &); + const int &, const int &, double &, const F90goms &, + const void*); // ----------------------------------------------------------------------------- } // extern C diff --git a/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.cc b/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.cc index aa9b92fb3..00fc6d60f 100644 --- a/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.cc +++ b/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.cc @@ -32,12 +32,9 @@ ObsRadianceRTTOVTLAD::ObsRadianceRTTOVTLAD(const ioda::ObsSpace & odb, const oops::ObsVariables & observed = odb.assimvariables(); std::vector channels_list = observed.channels(); - // values for passed qc - std::vector qc_passed = {QCflags::pass}; // call Fortran setup routine ufo_radiancerttov_tlad_setup_f90(keyOperRadianceRTTOV_, parameters.toConfiguration(), - channels_list.size(), channels_list[0], varin_, - qc_passed.size(), qc_passed[0]); + channels_list.size(), channels_list[0], varin_); oops::Log::trace() << "ObsRadianceRTTOVTLAD constructor done" << std::endl; } @@ -54,7 +51,7 @@ ObsRadianceRTTOVTLAD::~ObsRadianceRTTOVTLAD() { void ObsRadianceRTTOVTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics & ydiags, const QCFlags_t & qc_flags) { ufo_radiancerttov_tlad_settraj_f90(keyOperRadianceRTTOV_, geovals.toFortran(), obsspace(), - ydiags.toFortran(), qc_flags); + ydiags.toFortran()); oops::Log::trace() << "ObsRadianceRTTOVTLAD::setTrajectory done" << std::endl; } diff --git a/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.h b/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.h index 68800e777..a07c2ad1c 100644 --- a/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.h +++ b/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.h @@ -15,7 +15,6 @@ #include "oops/base/Variables.h" #include "oops/util/ObjectCounter.h" -#include "ufo/filters/QCflags.h" #include "ufo/LinearObsOperatorBase.h" #include "ufo/operators/rttov/Fortran/ObsRadianceRTTOVParameters.h" #include "ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.interface.h" diff --git a/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.interface.h b/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.interface.h index d24e54011..d91a3b9dc 100644 --- a/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.interface.h +++ b/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.interface.h @@ -27,15 +27,15 @@ extern "C" { // ----------------------------------------------------------------------------- void ufo_radiancerttov_tlad_setup_f90(F90hop &, const eckit::Configuration &, - const int &, const int &, - oops::Variables &, const int &, const int &); + const int &, const int &, + oops::Variables &); void ufo_radiancerttov_tlad_delete_f90(F90hop &); void ufo_radiancerttov_tlad_settraj_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, - const F90goms &, const ioda::ObsDataVector &); + const F90goms &); void ufo_radiancerttov_simobs_tl_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, - const int &, const int &, double &); + const int &, const int &, double &); void ufo_radiancerttov_simobs_ad_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, - const int &, const int &, const double &); + const int &, const int &, const double &); // ----------------------------------------------------------------------------- } // extern C diff --git a/src/ufo/profile/ProfileAveragePressure.cc b/src/ufo/profile/ProfileAveragePressure.cc index 9afbc4dad..ef7f616fa 100644 --- a/src/ufo/profile/ProfileAveragePressure.cc +++ b/src/ufo/profile/ProfileAveragePressure.cc @@ -106,13 +106,13 @@ namespace ufo { // ensure that all of the variables defined above have been added to the ObsSpace // on each rank. This prevents a hang when saving the ObsSpace. for (const auto & variableInt : variableNamesInt) { - profileDataHandler.get(variableInt); + const auto & vectorInt = profileDataHandler.get(variableInt); } for (const auto & variableFloat : variableNamesFloat) { - profileDataHandler.get(variableFloat); + const auto & vectorFloat = profileDataHandler.get(variableFloat); } for (const auto & variableBool : variableNamesBool) { - profileDataHandler.get(variableBool); + const auto & vectorBool = profileDataHandler.get(variableBool); } std::vector profiles = diff --git a/src/ufo/profile/ProfileAverageRelativeHumidity.cc b/src/ufo/profile/ProfileAverageRelativeHumidity.cc index 94c1ea523..10f6a32e8 100644 --- a/src/ufo/profile/ProfileAverageRelativeHumidity.cc +++ b/src/ufo/profile/ProfileAverageRelativeHumidity.cc @@ -70,13 +70,13 @@ namespace ufo { // ensure that all of the variables defined above have been added to the ObsSpace // on each rank. This prevents a hang when saving the ObsSpace. for (const auto & variableInt : variableNamesInt) { - profileDataHandler.get(variableInt); + const auto & vectorInt = profileDataHandler.get(variableInt); } for (const auto & variableFloat : variableNamesFloat) { - profileDataHandler.get(variableFloat); + const auto & vectorFloat = profileDataHandler.get(variableFloat); } for (const auto & variableBool : variableNamesBool) { - profileDataHandler.get(variableBool); + const auto & vectorBool = profileDataHandler.get(variableBool); } std::vector profiles = diff --git a/src/ufo/profile/ProfileAverageTemperature.cc b/src/ufo/profile/ProfileAverageTemperature.cc index e31c1305d..a849f47b3 100644 --- a/src/ufo/profile/ProfileAverageTemperature.cc +++ b/src/ufo/profile/ProfileAverageTemperature.cc @@ -80,13 +80,13 @@ namespace ufo { // ensure that all of the variables defined above have been added to the ObsSpace // on each rank. This prevents a hang when saving the ObsSpace. for (const auto & variableInt : variableNamesInt) { - profileDataHandler.get(variableInt); + const auto & vectorInt = profileDataHandler.get(variableInt); } for (const auto & variableFloat : variableNamesFloat) { - profileDataHandler.get(variableFloat); + const auto & vectorFloat = profileDataHandler.get(variableFloat); } for (const auto & variableBool : variableNamesBool) { - profileDataHandler.get(variableBool); + const auto & vectorBool = profileDataHandler.get(variableBool); } std::vector profiles = diff --git a/src/ufo/profile/ProfileAverageWindSpeed.cc b/src/ufo/profile/ProfileAverageWindSpeed.cc index b758df1ee..04d7646a2 100644 --- a/src/ufo/profile/ProfileAverageWindSpeed.cc +++ b/src/ufo/profile/ProfileAverageWindSpeed.cc @@ -80,13 +80,13 @@ namespace ufo { // ensure that all of the variables defined above have been added to the ObsSpace // on each rank. This prevents a hang when saving the ObsSpace. for (const auto & variableInt : variableNamesInt) { - profileDataHandler.get(variableInt); + const auto & vectorInt = profileDataHandler.get(variableInt); } for (const auto & variableFloat : variableNamesFloat) { - profileDataHandler.get(variableFloat); + const auto & vectorFloat = profileDataHandler.get(variableFloat); } for (const auto & variableBool : variableNamesBool) { - profileDataHandler.get(variableBool); + const auto & vectorBool = profileDataHandler.get(variableBool); } std::vector profiles = diff --git a/src/ufo/profile/ProfileCheckRH.cc b/src/ufo/profile/ProfileCheckRH.cc index 7de85bebc..29b17e6e7 100644 --- a/src/ufo/profile/ProfileCheckRH.cc +++ b/src/ufo/profile/ProfileCheckRH.cc @@ -36,6 +36,7 @@ namespace ufo { profileDataHandler.get(ufo::ProfileVariableNames::obs_dew_point_temperature); const std::vector &diagFlagsTTropopause = profileDataHandler.get(ufo::ProfileVariableNames::diagflags_tropo_t); + const std::vector &diagFlagsTFinalReject = profileDataHandler.get(ufo::ProfileVariableNames::diagflags_final_reject_t); std::vector &diagFlagsRHInterp = profileDataHandler.get(ufo::ProfileVariableNames::diagflags_interpolation_rh); diff --git a/src/ufo/utils/SaveBiasCoeffs.cc b/src/ufo/utils/SaveBiasCoeffs.cc index aeaa8d3d3..6489f9251 100644 --- a/src/ufo/utils/SaveBiasCoeffs.cc +++ b/src/ufo/utils/SaveBiasCoeffs.cc @@ -42,7 +42,7 @@ void saveBiasCoeffsWithChannels(ioda::Group & parent, ioda::Variable biasVar = ogrp.vars.createWithScales( "BiasCoefficients/"+predictors[jpred], {ogrp.vars["Record"], ogrp.vars["Channel"]}, float_params); - biasVar.writeWithEigenRegular(coeffs(jpred, Eigen::placeholders::all)); + biasVar.writeWithEigenRegular(coeffs(jpred, Eigen::all)); } // Save the Channel @@ -78,7 +78,7 @@ void saveBiasCoeffsWithRecords(ioda::Group & parent, ioda::Variable biasVar = ogrp.vars.createWithScales( "BiasCoefficients/"+predictors[jpred], {ogrp.vars["Record"], ogrp.vars["Variable"]}, float_params); - biasVar.writeWithEigenRegular(coeffs(jpred, Eigen::placeholders::all)); + biasVar.writeWithEigenRegular(coeffs(jpred, Eigen::all)); } // Save the stationIdentification diff --git a/src/ufo/utils/SaveBiasCovariance.cc b/src/ufo/utils/SaveBiasCovariance.cc index 2fec25d6b..c0850ba6b 100644 --- a/src/ufo/utils/SaveBiasCovariance.cc +++ b/src/ufo/utils/SaveBiasCovariance.cc @@ -41,7 +41,7 @@ void saveBiasCovarianceWithChannels(ioda::Group & parent, ioda::Variable anvarVar = ogrp.vars.createWithScales( "BiasCoefficientErrors/"+predictors[jpred], {ogrp.vars["Record"], ogrp.vars["Channel"]}, float_params); - anvarVar.writeWithEigenRegular(allbcerrors(jpred, Eigen::placeholders::all)); + anvarVar.writeWithEigenRegular(allbcerrors(jpred, Eigen::all)); } // write number_obs_assimilated @@ -82,7 +82,7 @@ void saveBiasCovarianceWithRecords(ioda::Group & parent, ioda::Variable biasVar = ogrp.vars.createWithScales( "BiasCoefficientErrors/" + predictors[jpred], {ogrp.vars["Record"], ogrp.vars["Variable"]}, float_params); - biasVar.writeWithEigenRegular(allbcerrors(jpred, Eigen::placeholders::all)); + biasVar.writeWithEigenRegular(allbcerrors(jpred, Eigen::all)); } // write number_obs_assimilated diff --git a/src/ufo/utils/dataextractor/DataExtractor.h b/src/ufo/utils/dataextractor/DataExtractor.h index d28c5551b..e51e6daad 100644 --- a/src/ufo/utils/dataextractor/DataExtractor.h +++ b/src/ufo/utils/dataextractor/DataExtractor.h @@ -15,13 +15,7 @@ #include #include -// To skip the warning about maybe-uninitialized variables in boost multi_array headers. -// Will only skip this warning for the boost multi_array headers, -// and only for this file so that we can still get the warning for our own code. -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" #include -#pragma GCC diagnostic pop #include #include #include diff --git a/src/ufo/utils/dataextractor/DataExtractorNetCDFBackend.h b/src/ufo/utils/dataextractor/DataExtractorNetCDFBackend.h index 855d9d24c..96dddbd8d 100644 --- a/src/ufo/utils/dataextractor/DataExtractorNetCDFBackend.h +++ b/src/ufo/utils/dataextractor/DataExtractorNetCDFBackend.h @@ -88,7 +88,7 @@ namespace ufo /// group: ErrorVariance { /// variables: /// float airTemperature(sensorChannelNumber, MetaData/sensorChannelNumber, index) ; -/// airTemperature:coordinates = "MetaData/latitudeBand +/// airTemperature:coordinates = "MetaData/latitudeBand \ /// MetaData/processingCenter MetaData/satelliteIdentifier" ; /// } /// \endcode diff --git a/src/ufo/variabletransforms/Cal_AdjustedHeightCoordinate.cc b/src/ufo/variabletransforms/Cal_AdjustedHeightCoordinate.cc index ce3e45b19..2a64d119d 100644 --- a/src/ufo/variabletransforms/Cal_AdjustedHeightCoordinate.cc +++ b/src/ufo/variabletransforms/Cal_AdjustedHeightCoordinate.cc @@ -28,8 +28,8 @@ namespace ufo { Cal_AdjustedHeightCoordinate::Cal_AdjustedHeightCoordinate( const GenericVariableTransformParameters & options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) : + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), gvals_() { oops::Log::trace() << "Cal_AdjustedHeightCoordinate::Constructor start" << std::endl; // List of GeoVaLs this transform will need access to diff --git a/src/ufo/variabletransforms/Cal_AdjustedHeightCoordinate.h b/src/ufo/variabletransforms/Cal_AdjustedHeightCoordinate.h index 71c05361d..4dfc8e5dd 100644 --- a/src/ufo/variabletransforms/Cal_AdjustedHeightCoordinate.h +++ b/src/ufo/variabletransforms/Cal_AdjustedHeightCoordinate.h @@ -30,9 +30,9 @@ namespace ufo { class Cal_AdjustedHeightCoordinate : public TransformBase { public: Cal_AdjustedHeightCoordinate(const GenericVariableTransformParameters &options, - const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const ObsFilterData &data, + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; Variables requiredVariables() const override { return gvals_; } diff --git a/src/ufo/variabletransforms/Cal_HeightFromPressure.cc b/src/ufo/variabletransforms/Cal_HeightFromPressure.cc index 192255abb..8ed3bd197 100644 --- a/src/ufo/variabletransforms/Cal_HeightFromPressure.cc +++ b/src/ufo/variabletransforms/Cal_HeightFromPressure.cc @@ -18,8 +18,8 @@ static TransformMaker Cal_HeightFromPressure::Cal_HeightFromPressure( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), heightCoord_(options.heightCoord), heightGroup_(options.heightGroup), diff --git a/src/ufo/variabletransforms/Cal_HeightFromPressure.h b/src/ufo/variabletransforms/Cal_HeightFromPressure.h index a19cb26c9..073811d9e 100644 --- a/src/ufo/variabletransforms/Cal_HeightFromPressure.h +++ b/src/ufo/variabletransforms/Cal_HeightFromPressure.h @@ -44,8 +44,8 @@ class Cal_HeightFromPressure : public TransformBase { Cal_HeightFromPressure(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run check void runTransform(const std::vector &apply) override; diff --git a/src/ufo/variabletransforms/Cal_Humidity.cc b/src/ufo/variabletransforms/Cal_Humidity.cc index 8f57ce994..988154ab4 100644 --- a/src/ufo/variabletransforms/Cal_Humidity.cc +++ b/src/ufo/variabletransforms/Cal_Humidity.cc @@ -19,8 +19,8 @@ static TransformMaker Cal_RelativeHumidity::Cal_RelativeHumidity( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), allowSuperSaturation_(options.AllowSuperSaturation), specifichumidityvariable_(options.SpecificHumidityVariable), @@ -427,8 +427,8 @@ static TransformMaker Cal_SpecificHumidity::Cal_SpecificHumidity( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), specifichumidityvariable_(options.SpecificHumidityVariable), pressurevariable_(options.PressureVariable), @@ -651,8 +651,8 @@ static TransformMaker Cal_VirtualTemperature::Cal_VirtualTemperature( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), specifichumidityvariable_(options.SpecificHumidityVariable), temperaturevariable_(options.TemperatureVariable), diff --git a/src/ufo/variabletransforms/Cal_Humidity.h b/src/ufo/variabletransforms/Cal_Humidity.h index 268115267..754493ed7 100644 --- a/src/ufo/variabletransforms/Cal_Humidity.h +++ b/src/ufo/variabletransforms/Cal_Humidity.h @@ -79,8 +79,8 @@ class Cal_RelativeHumidity : public TransformBase { Cal_RelativeHumidity(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; @@ -130,8 +130,8 @@ class Cal_SpecificHumidity : public TransformBase { Cal_SpecificHumidity(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run check void runTransform(const std::vector &apply) override; @@ -172,8 +172,8 @@ class Cal_VirtualTemperature : public TransformBase { Cal_VirtualTemperature(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run check void runTransform(const std::vector &apply) override; diff --git a/src/ufo/variabletransforms/Cal_IceThicknessFromFreeboard.cc b/src/ufo/variabletransforms/Cal_IceThicknessFromFreeboard.cc index 18a5a3188..80ac71800 100644 --- a/src/ufo/variabletransforms/Cal_IceThicknessFromFreeboard.cc +++ b/src/ufo/variabletransforms/Cal_IceThicknessFromFreeboard.cc @@ -26,8 +26,8 @@ makerCal_IceThicknessFromFreeboard_("Calculate iceThickness from seaIceFreeboard Cal_IceThicknessFromFreeboard::Cal_IceThicknessFromFreeboard( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), options_(options) {} // ----------------------------------------------------------------------------- diff --git a/src/ufo/variabletransforms/Cal_IceThicknessFromFreeboard.h b/src/ufo/variabletransforms/Cal_IceThicknessFromFreeboard.h index 080ae9ee8..5490a0617 100644 --- a/src/ufo/variabletransforms/Cal_IceThicknessFromFreeboard.h +++ b/src/ufo/variabletransforms/Cal_IceThicknessFromFreeboard.h @@ -135,8 +135,8 @@ class Cal_IceThicknessFromFreeboard : public TransformBase { typedef Cal_IceThicknessFromFreeboardParameters Parameters_; Cal_IceThicknessFromFreeboard( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; diff --git a/src/ufo/variabletransforms/Cal_Logarithm.cc b/src/ufo/variabletransforms/Cal_Logarithm.cc index fa76c1238..16516173f 100644 --- a/src/ufo/variabletransforms/Cal_Logarithm.cc +++ b/src/ufo/variabletransforms/Cal_Logarithm.cc @@ -18,8 +18,8 @@ static TransformMaker makerCal_Logarithm_("Logarithm"); Cal_Logarithm::Cal_Logarithm( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), variable_(options.variable), group_(options.group) { diff --git a/src/ufo/variabletransforms/Cal_Logarithm.h b/src/ufo/variabletransforms/Cal_Logarithm.h index 53ffb6e4a..306cdecd0 100644 --- a/src/ufo/variabletransforms/Cal_Logarithm.h +++ b/src/ufo/variabletransforms/Cal_Logarithm.h @@ -47,8 +47,8 @@ class Cal_Logarithm : public TransformBase { typedef Cal_LogarithmParameters Parameters_; Cal_Logarithm(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run check void runTransform(const std::vector &apply) override; diff --git a/src/ufo/variabletransforms/Cal_PStar.cc b/src/ufo/variabletransforms/Cal_PStar.cc index b5c70184e..d9ad4ca4f 100644 --- a/src/ufo/variabletransforms/Cal_PStar.cc +++ b/src/ufo/variabletransforms/Cal_PStar.cc @@ -23,8 +23,8 @@ static TransformMaker Cal_PStar::Cal_PStar( const GenericVariableTransformParameters &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), gvals_() { gvals_ += Variable("surf_param_a@GeoVaLs"); gvals_ += Variable("surf_param_b@GeoVaLs"); diff --git a/src/ufo/variabletransforms/Cal_PStar.h b/src/ufo/variabletransforms/Cal_PStar.h index 3e90482a5..88752a071 100644 --- a/src/ufo/variabletransforms/Cal_PStar.h +++ b/src/ufo/variabletransforms/Cal_PStar.h @@ -44,8 +44,8 @@ class Cal_PStar : public TransformBase { public: Cal_PStar(const GenericVariableTransformParameters &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; Variables requiredVariables() const override { return gvals_; } diff --git a/src/ufo/variabletransforms/Cal_PotentialTFromT.cc b/src/ufo/variabletransforms/Cal_PotentialTFromT.cc index 30337a1f3..28cfc7953 100644 --- a/src/ufo/variabletransforms/Cal_PotentialTFromT.cc +++ b/src/ufo/variabletransforms/Cal_PotentialTFromT.cc @@ -22,8 +22,8 @@ static TransformMaker Cal_PotentialTFromT::Cal_PotentialTFromT( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), pressurevariable_(options.PressureVariable), pressuregroup_(options.PressureGroup), diff --git a/src/ufo/variabletransforms/Cal_PotentialTFromT.h b/src/ufo/variabletransforms/Cal_PotentialTFromT.h index 14590f79f..e056d8f36 100644 --- a/src/ufo/variabletransforms/Cal_PotentialTFromT.h +++ b/src/ufo/variabletransforms/Cal_PotentialTFromT.h @@ -59,8 +59,8 @@ class Cal_PotentialTFromT : public TransformBase { Cal_PotentialTFromT(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; diff --git a/src/ufo/variabletransforms/Cal_PressureFromHeight.cc b/src/ufo/variabletransforms/Cal_PressureFromHeight.cc index ee9e002cf..fe891bffe 100644 --- a/src/ufo/variabletransforms/Cal_PressureFromHeight.cc +++ b/src/ufo/variabletransforms/Cal_PressureFromHeight.cc @@ -18,8 +18,8 @@ static TransformMaker Cal_PressureFromHeightForProfile::Cal_PressureFromHeightForProfile( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), heightCoord_(options.HeightCoord), pressureCoord_(options.PressureCoord), @@ -236,8 +236,8 @@ static TransformMaker Cal_PressureFromHeightForICAO::Cal_PressureFromHeightForICAO( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), heightCoord_(options.HeightCoord), pressureCoord_(options.PressureCoord), diff --git a/src/ufo/variabletransforms/Cal_PressureFromHeight.h b/src/ufo/variabletransforms/Cal_PressureFromHeight.h index 3acc48a0a..d63ae169c 100644 --- a/src/ufo/variabletransforms/Cal_PressureFromHeight.h +++ b/src/ufo/variabletransforms/Cal_PressureFromHeight.h @@ -48,8 +48,8 @@ class Cal_PressureFromHeightForProfile : public TransformBase { Cal_PressureFromHeightForProfile(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; @@ -80,8 +80,8 @@ class Cal_PressureFromHeightForICAO : public TransformBase { Cal_PressureFromHeightForICAO(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run check void runTransform(const std::vector &apply) override; diff --git a/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.cc b/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.cc index df58c73ac..11d4c0d96 100644 --- a/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.cc +++ b/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.cc @@ -20,8 +20,8 @@ makerCal_ProfileHorizontalDrift_("ProfileHorizontalDrift"); Cal_ProfileHorizontalDrift::Cal_ProfileHorizontalDrift (const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), heightCoord_(options.HeightCoord), keep_in_window_(options.keep_in_window), diff --git a/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.h b/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.h index 0d9ef53da..dd39a0c7c 100644 --- a/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.h +++ b/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.h @@ -60,8 +60,8 @@ class Cal_ProfileHorizontalDrift : public TransformBase { Cal_ProfileHorizontalDrift(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; diff --git a/src/ufo/variabletransforms/Cal_QNHtoQFEpressure.cc b/src/ufo/variabletransforms/Cal_QNHtoQFEpressure.cc index 08ed910ca..44f374697 100644 --- a/src/ufo/variabletransforms/Cal_QNHtoQFEpressure.cc +++ b/src/ufo/variabletransforms/Cal_QNHtoQFEpressure.cc @@ -21,8 +21,8 @@ static TransformMaker Cal_QNHtoQFEpressure::Cal_QNHtoQFEpressure( const GenericVariableTransformParameters &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr) {} /************************************************************************************/ diff --git a/src/ufo/variabletransforms/Cal_QNHtoQFEpressure.h b/src/ufo/variabletransforms/Cal_QNHtoQFEpressure.h index bb63ecdd3..dbe0e5638 100644 --- a/src/ufo/variabletransforms/Cal_QNHtoQFEpressure.h +++ b/src/ufo/variabletransforms/Cal_QNHtoQFEpressure.h @@ -36,8 +36,8 @@ class Cal_QNHtoQFEpressure : public TransformBase { public: Cal_QNHtoQFEpressure(const GenericVariableTransformParameters &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; }; diff --git a/src/ufo/variabletransforms/Cal_RadarBeamGeometry.cc b/src/ufo/variabletransforms/Cal_RadarBeamGeometry.cc index c640bfb7b..b239685ee 100644 --- a/src/ufo/variabletransforms/Cal_RadarBeamGeometry.cc +++ b/src/ufo/variabletransforms/Cal_RadarBeamGeometry.cc @@ -20,8 +20,8 @@ static TransformMaker makerCal_RadarBeamGeometry_("RadarB Cal_RadarBeamGeometry::Cal_RadarBeamGeometry( const Parameters_ & options, const ObsFilterData & data, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + const std::shared_ptr> & flags, + const std::shared_ptr> & obserr) : TransformBase(options, data, flags, obserr), params_(options) {} diff --git a/src/ufo/variabletransforms/Cal_RadarBeamGeometry.h b/src/ufo/variabletransforms/Cal_RadarBeamGeometry.h index 8c6f4dcbe..3d626cfd5 100644 --- a/src/ufo/variabletransforms/Cal_RadarBeamGeometry.h +++ b/src/ufo/variabletransforms/Cal_RadarBeamGeometry.h @@ -63,8 +63,8 @@ class Cal_RadarBeamGeometry : public TransformBase { Cal_RadarBeamGeometry(const Parameters_ & options, const ObsFilterData & data, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + const std::shared_ptr> & flags, + const std::shared_ptr> & obserr); void runTransform(const std::vector &apply) override; private: diff --git a/src/ufo/variabletransforms/Cal_RemapScanPosition.cc b/src/ufo/variabletransforms/Cal_RemapScanPosition.cc index 5025c7382..36bd3afe7 100644 --- a/src/ufo/variabletransforms/Cal_RemapScanPosition.cc +++ b/src/ufo/variabletransforms/Cal_RemapScanPosition.cc @@ -19,8 +19,8 @@ static TransformMaker Cal_RemapScanPosition::Cal_RemapScanPosition( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), parameters_(options) {} /************************************************************************************/ diff --git a/src/ufo/variabletransforms/Cal_RemapScanPosition.h b/src/ufo/variabletransforms/Cal_RemapScanPosition.h index be06f3cd2..e97f86a16 100644 --- a/src/ufo/variabletransforms/Cal_RemapScanPosition.h +++ b/src/ufo/variabletransforms/Cal_RemapScanPosition.h @@ -52,8 +52,8 @@ class Cal_RemapScanPosition : public TransformBase { Cal_RemapScanPosition(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; private: diff --git a/src/ufo/variabletransforms/Cal_SatBrightnessTempFromRad.cc b/src/ufo/variabletransforms/Cal_SatBrightnessTempFromRad.cc index 6474f6507..1774b9009 100644 --- a/src/ufo/variabletransforms/Cal_SatBrightnessTempFromRad.cc +++ b/src/ufo/variabletransforms/Cal_SatBrightnessTempFromRad.cc @@ -25,8 +25,8 @@ static TransformMaker Cal_SatBrightnessTempFromRad::Cal_SatBrightnessTempFromRad( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), parameters_(options), variables_(), channels_(parameters_.transformVariable.value().channels()) { variables_ += parameters_.transformVariable.value(); diff --git a/src/ufo/variabletransforms/Cal_SatBrightnessTempFromRad.h b/src/ufo/variabletransforms/Cal_SatBrightnessTempFromRad.h index 22f83a204..46ef1c54d 100644 --- a/src/ufo/variabletransforms/Cal_SatBrightnessTempFromRad.h +++ b/src/ufo/variabletransforms/Cal_SatBrightnessTempFromRad.h @@ -127,8 +127,8 @@ class Cal_SatBrightnessTempFromRad : public TransformBase { Cal_SatBrightnessTempFromRad(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; Variables requiredVariables() const override { return variables_; } diff --git a/src/ufo/variabletransforms/Cal_SatRadianceFromPCScores.cc b/src/ufo/variabletransforms/Cal_SatRadianceFromPCScores.cc index 730ebf364..6a495d685 100644 --- a/src/ufo/variabletransforms/Cal_SatRadianceFromPCScores.cc +++ b/src/ufo/variabletransforms/Cal_SatRadianceFromPCScores.cc @@ -29,8 +29,8 @@ static TransformMaker Cal_SatRadianceFromPCScores::Cal_SatRadianceFromPCScores( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), parameters_(options), variables_({parameters_.pcVariable.value()}), channels_(parameters_.destinationVariable.value().channels()) { @@ -128,7 +128,7 @@ void Cal_SatRadianceFromPCScores::runTransform(const std::vector &apply) { // multiplying also by a scaling factor to produce radiances in W m^-2 sr^-1 (m^-1)^-1 float scalingFactor = parameters_.scalingFactor.value(); Eigen::MatrixXf reconMatrix \ - = reconstructor(Eigen::placeholders::all, destinationChannelIndex).transpose() * pcScoresMatrix; + = reconstructor(Eigen::all, destinationChannelIndex).transpose() * pcScoresMatrix; reconMatrix *= scalingFactor; // Loop over channels and obs locations to populate derived observation variable diff --git a/src/ufo/variabletransforms/Cal_SatRadianceFromPCScores.h b/src/ufo/variabletransforms/Cal_SatRadianceFromPCScores.h index 1eeda28d8..e4481e60e 100644 --- a/src/ufo/variabletransforms/Cal_SatRadianceFromPCScores.h +++ b/src/ufo/variabletransforms/Cal_SatRadianceFromPCScores.h @@ -126,9 +126,9 @@ class Cal_SatRadianceFromPCScores : public TransformBase { typedef Eigen::Matrix rowMajorMatrix; Cal_SatRadianceFromPCScores(const Parameters_ &options, - const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const ObsFilterData &data, + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; Variables requiredVariables() const override { return variables_; } diff --git a/src/ufo/variabletransforms/Cal_SatRadianceFromScaledRadiance.cc b/src/ufo/variabletransforms/Cal_SatRadianceFromScaledRadiance.cc index c0156b0e9..7257aa5b7 100644 --- a/src/ufo/variabletransforms/Cal_SatRadianceFromScaledRadiance.cc +++ b/src/ufo/variabletransforms/Cal_SatRadianceFromScaledRadiance.cc @@ -25,8 +25,8 @@ static TransformMaker Cal_SatRadianceFromScaledRadiance::Cal_SatRadianceFromScaledRadiance( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), parameters_(options), variables_({parameters_.transformVariable.value()}), channels_(parameters_.transformVariable.value().channels()) { @@ -105,8 +105,8 @@ void Cal_SatRadianceFromScaledRadiance::runTransform(const std::vector &ap if (radiance[ichan][iloc] != missingValueFloat && radiance[ichan][iloc] > 0.0f) { for (size_t iscale = 0; iscale < numScaleFactors; ++iscale) { - if ( (channels_[ichan] >= startChannelScale[iscale]) && - (channels_[ichan] <= endChannelScale[iscale]) ) { + if (channels_[ichan] >= startChannelScale[iscale] & + channels_[ichan] <= endChannelScale[iscale]) { radiance[ichan][iloc] *= std::pow(10, (-1.0f*channelScaleFactor[iscale])); break; } // if channels_ diff --git a/src/ufo/variabletransforms/Cal_SatRadianceFromScaledRadiance.h b/src/ufo/variabletransforms/Cal_SatRadianceFromScaledRadiance.h index 741c3c2a6..10d3e46b8 100644 --- a/src/ufo/variabletransforms/Cal_SatRadianceFromScaledRadiance.h +++ b/src/ufo/variabletransforms/Cal_SatRadianceFromScaledRadiance.h @@ -86,9 +86,9 @@ class Cal_SatRadianceFromScaledRadiance : public TransformBase { typedef Cal_SatRadianceFromScaledRadianceParameters Parameters_; Cal_SatRadianceFromScaledRadiance(const Parameters_ &options, - const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const ObsFilterData &data, + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; Variables requiredVariables() const override { return variables_; } diff --git a/src/ufo/variabletransforms/Cal_SatZenithAngleCorrection.cc b/src/ufo/variabletransforms/Cal_SatZenithAngleCorrection.cc index cf5ec4aa4..75a560374 100644 --- a/src/ufo/variabletransforms/Cal_SatZenithAngleCorrection.cc +++ b/src/ufo/variabletransforms/Cal_SatZenithAngleCorrection.cc @@ -24,8 +24,8 @@ static TransformMaker Cal_SatZenithAngleCorrection::Cal_SatZenithAngleCorrection( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), parameters_(options), variables_({parameters_.transformVariable.value()}) { ASSERT(parameters_.coeffA.value().size() == parameters_.transformVariable.value().size()); diff --git a/src/ufo/variabletransforms/Cal_SatZenithAngleCorrection.h b/src/ufo/variabletransforms/Cal_SatZenithAngleCorrection.h index 8a7f2f7ef..9f36f59db 100644 --- a/src/ufo/variabletransforms/Cal_SatZenithAngleCorrection.h +++ b/src/ufo/variabletransforms/Cal_SatZenithAngleCorrection.h @@ -92,8 +92,8 @@ class Cal_SatZenithAngleCorrection : public TransformBase { Cal_SatZenithAngleCorrection(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; Variables requiredVariables() const override { return variables_; } diff --git a/src/ufo/variabletransforms/Cal_SurfaceWindScalingCombined.cc b/src/ufo/variabletransforms/Cal_SurfaceWindScalingCombined.cc index 48c154fd5..f8548b89d 100644 --- a/src/ufo/variabletransforms/Cal_SurfaceWindScalingCombined.cc +++ b/src/ufo/variabletransforms/Cal_SurfaceWindScalingCombined.cc @@ -27,8 +27,8 @@ namespace ufo { Cal_SurfaceWindScalingCombined::Cal_SurfaceWindScalingCombined( const GenericVariableTransformParameters &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) : + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), gvals_() { oops::Log::trace() << "Cal_SurfaceWindScalingCombined::Constructor start" << std::endl; oops::Log::trace() << "Cal_SurfaceWindScalingCombined::Constructor done" << std::endl; diff --git a/src/ufo/variabletransforms/Cal_SurfaceWindScalingCombined.h b/src/ufo/variabletransforms/Cal_SurfaceWindScalingCombined.h index fcb5ea266..3fd418280 100644 --- a/src/ufo/variabletransforms/Cal_SurfaceWindScalingCombined.h +++ b/src/ufo/variabletransforms/Cal_SurfaceWindScalingCombined.h @@ -31,9 +31,9 @@ namespace ufo { class Cal_SurfaceWindScalingCombined : public TransformBase { public: Cal_SurfaceWindScalingCombined(const GenericVariableTransformParameters &options, - const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const ObsFilterData &data, + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; Variables requiredVariables() const override { return gvals_; } diff --git a/src/ufo/variabletransforms/Cal_SurfaceWindScalingHeight.cc b/src/ufo/variabletransforms/Cal_SurfaceWindScalingHeight.cc index ddeeff99d..384d05af2 100644 --- a/src/ufo/variabletransforms/Cal_SurfaceWindScalingHeight.cc +++ b/src/ufo/variabletransforms/Cal_SurfaceWindScalingHeight.cc @@ -28,8 +28,8 @@ namespace ufo { Cal_SurfaceWindScalingHeight::Cal_SurfaceWindScalingHeight( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) : + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), gvals_(), heightVariableGroup_(options.heightVariableGroup), heightVariableName_(options.heightVariableName) diff --git a/src/ufo/variabletransforms/Cal_SurfaceWindScalingHeight.h b/src/ufo/variabletransforms/Cal_SurfaceWindScalingHeight.h index 262193b8b..a24c21b02 100644 --- a/src/ufo/variabletransforms/Cal_SurfaceWindScalingHeight.h +++ b/src/ufo/variabletransforms/Cal_SurfaceWindScalingHeight.h @@ -47,8 +47,8 @@ class Cal_SurfaceWindScalingHeightParameters: public VariableTransformParameters typedef Cal_SurfaceWindScalingHeightParameters Parameters_; Cal_SurfaceWindScalingHeight(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; Variables requiredVariables() const override { return gvals_; } diff --git a/src/ufo/variabletransforms/Cal_SurfaceWindScalingPressure.cc b/src/ufo/variabletransforms/Cal_SurfaceWindScalingPressure.cc index d7a173b3e..66a1a1b6d 100644 --- a/src/ufo/variabletransforms/Cal_SurfaceWindScalingPressure.cc +++ b/src/ufo/variabletransforms/Cal_SurfaceWindScalingPressure.cc @@ -28,8 +28,8 @@ namespace ufo { Cal_SurfaceWindScalingPressure::Cal_SurfaceWindScalingPressure( const GenericVariableTransformParameters &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) : + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), gvals_() { oops::Log::trace() << "Cal_SurfaceWindScalingPressure::Constructor start" << std::endl; // List of GeoVaLs this transform will need access to diff --git a/src/ufo/variabletransforms/Cal_SurfaceWindScalingPressure.h b/src/ufo/variabletransforms/Cal_SurfaceWindScalingPressure.h index 81d49f4b2..5e91509ac 100644 --- a/src/ufo/variabletransforms/Cal_SurfaceWindScalingPressure.h +++ b/src/ufo/variabletransforms/Cal_SurfaceWindScalingPressure.h @@ -31,9 +31,9 @@ namespace ufo { class Cal_SurfaceWindScalingPressure : public TransformBase { public: Cal_SurfaceWindScalingPressure(const GenericVariableTransformParameters &options, - const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const ObsFilterData &data, + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; Variables requiredVariables() const override { return gvals_; } diff --git a/src/ufo/variabletransforms/Cal_Wind.cc b/src/ufo/variabletransforms/Cal_Wind.cc index d6451007a..7db431d4e 100644 --- a/src/ufo/variabletransforms/Cal_Wind.cc +++ b/src/ufo/variabletransforms/Cal_Wind.cc @@ -23,8 +23,8 @@ static TransformMaker Cal_WindSpeedAndDirection::Cal_WindSpeedAndDirection( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), group_(options.group), eastwardwindvariable_(options.EastwardWindVariable), @@ -89,8 +89,8 @@ static TransformMaker Cal_WindComponents::Cal_WindComponents( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), group_(options.group), windspeedvariable_(options.WindSpeedVariable), diff --git a/src/ufo/variabletransforms/Cal_Wind.h b/src/ufo/variabletransforms/Cal_Wind.h index c3820cb5b..4660d0d3b 100644 --- a/src/ufo/variabletransforms/Cal_Wind.h +++ b/src/ufo/variabletransforms/Cal_Wind.h @@ -60,8 +60,8 @@ class Cal_WindSpeedAndDirection : public TransformBase { Cal_WindSpeedAndDirection(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; @@ -90,16 +90,16 @@ class Cal_WindComponents : public TransformBase { Cal_WindComponents(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; private: - /// Group name. - std::string group_; std::string windspeedvariable_; std::string winddirectionvariable_; + /// Group name. + std::string group_; }; } // namespace ufo diff --git a/src/ufo/variabletransforms/OceanConversions/OceanDensity.cc b/src/ufo/variabletransforms/OceanConversions/OceanDensity.cc index 828876497..ffb33c756 100644 --- a/src/ufo/variabletransforms/OceanConversions/OceanDensity.cc +++ b/src/ufo/variabletransforms/OceanConversions/OceanDensity.cc @@ -23,8 +23,8 @@ static TransformMaker OceanDensity::OceanDensity( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), salinityvariable_(options.SalinityVariable), salinitygroup_(options.SalinityGroup), diff --git a/src/ufo/variabletransforms/OceanConversions/OceanDensity.h b/src/ufo/variabletransforms/OceanConversions/OceanDensity.h index fac206d66..29037c9ae 100644 --- a/src/ufo/variabletransforms/OceanConversions/OceanDensity.h +++ b/src/ufo/variabletransforms/OceanConversions/OceanDensity.h @@ -77,8 +77,8 @@ class OceanDensity : public TransformBase { typedef OceanDensityParameters Parameters_; OceanDensity(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; diff --git a/src/ufo/variabletransforms/OceanConversions/OceanDepthToPressure.cc b/src/ufo/variabletransforms/OceanConversions/OceanDepthToPressure.cc index 7cbdf10ff..6f6db5842 100644 --- a/src/ufo/variabletransforms/OceanConversions/OceanDepthToPressure.cc +++ b/src/ufo/variabletransforms/OceanConversions/OceanDepthToPressure.cc @@ -23,8 +23,8 @@ static TransformMaker OceanDepthToPressure::OceanDepthToPressure( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), depthvariable_(options.DepthVariable), depthgroup_(options.DepthGroup), diff --git a/src/ufo/variabletransforms/OceanConversions/OceanDepthToPressure.h b/src/ufo/variabletransforms/OceanConversions/OceanDepthToPressure.h index 3f2ccadb9..40ea94a9d 100644 --- a/src/ufo/variabletransforms/OceanConversions/OceanDepthToPressure.h +++ b/src/ufo/variabletransforms/OceanConversions/OceanDepthToPressure.h @@ -64,8 +64,8 @@ class OceanDepthToPressure : public TransformBase { typedef OceanDepthToPressureParameters Parameters_; OceanDepthToPressure(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; diff --git a/src/ufo/variabletransforms/OceanConversions/OceanPracticalSalinityToAbsoluteSalinity.cc b/src/ufo/variabletransforms/OceanConversions/OceanPracticalSalinityToAbsoluteSalinity.cc index 0950fdfd5..83df41237 100644 --- a/src/ufo/variabletransforms/OceanConversions/OceanPracticalSalinityToAbsoluteSalinity.cc +++ b/src/ufo/variabletransforms/OceanConversions/OceanPracticalSalinityToAbsoluteSalinity.cc @@ -23,8 +23,8 @@ static TransformMaker OceanPracticalSalinityToAbsoluteSalinity::OceanPracticalSalinityToAbsoluteSalinity( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), absolutesalinityvariable_(options.AbsoluteSalinityVariable), practicalsalinityvariable_(options.PracticalSalinityVariable), diff --git a/src/ufo/variabletransforms/OceanConversions/OceanPracticalSalinityToAbsoluteSalinity.h b/src/ufo/variabletransforms/OceanConversions/OceanPracticalSalinityToAbsoluteSalinity.h index 2403a77b9..0ab5039ff 100644 --- a/src/ufo/variabletransforms/OceanConversions/OceanPracticalSalinityToAbsoluteSalinity.h +++ b/src/ufo/variabletransforms/OceanConversions/OceanPracticalSalinityToAbsoluteSalinity.h @@ -74,9 +74,9 @@ class OceanPracticalSalinityToAbsoluteSalinity : public TransformBase { public: typedef OceanPracticalSalinityToAbsoluteSalinityParameters Parameters_; OceanPracticalSalinityToAbsoluteSalinity(const Parameters_ &options, - const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const ObsFilterData &data, + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; diff --git a/src/ufo/variabletransforms/OceanConversions/OceanTempToConservativeTemp.cc b/src/ufo/variabletransforms/OceanConversions/OceanTempToConservativeTemp.cc index cef5da03e..8d168be7b 100644 --- a/src/ufo/variabletransforms/OceanConversions/OceanTempToConservativeTemp.cc +++ b/src/ufo/variabletransforms/OceanConversions/OceanTempToConservativeTemp.cc @@ -23,8 +23,8 @@ static TransformMaker OceanTempToConservativeTemp::OceanTempToConservativeTemp( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), salinityvariable_(options.SalinityVariable), salinitygroup_(options.SalinityGroup), diff --git a/src/ufo/variabletransforms/OceanConversions/OceanTempToConservativeTemp.h b/src/ufo/variabletransforms/OceanConversions/OceanTempToConservativeTemp.h index 1074b91ce..31a69fc70 100644 --- a/src/ufo/variabletransforms/OceanConversions/OceanTempToConservativeTemp.h +++ b/src/ufo/variabletransforms/OceanConversions/OceanTempToConservativeTemp.h @@ -79,8 +79,8 @@ class OceanTempToConservativeTemp : public TransformBase { typedef OceanTempToConservativeTempParameters Parameters_; OceanTempToConservativeTemp(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; diff --git a/src/ufo/variabletransforms/OceanConversions/OceanTempToTheta.cc b/src/ufo/variabletransforms/OceanConversions/OceanTempToTheta.cc index 3b0cfbee5..99ef95bdd 100644 --- a/src/ufo/variabletransforms/OceanConversions/OceanTempToTheta.cc +++ b/src/ufo/variabletransforms/OceanConversions/OceanTempToTheta.cc @@ -23,8 +23,8 @@ static TransformMaker OceanTempToTheta::OceanTempToTheta( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), salinityvariable_(options.SalinityVariable), salinitygroup_(options.SalinityGroup), diff --git a/src/ufo/variabletransforms/OceanConversions/OceanTempToTheta.h b/src/ufo/variabletransforms/OceanConversions/OceanTempToTheta.h index 73539b16a..9d8dbcb18 100644 --- a/src/ufo/variabletransforms/OceanConversions/OceanTempToTheta.h +++ b/src/ufo/variabletransforms/OceanConversions/OceanTempToTheta.h @@ -79,8 +79,8 @@ class OceanTempToTheta : public TransformBase { typedef OceanTempToThetaParameters Parameters_; OceanTempToTheta(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; diff --git a/src/ufo/variabletransforms/TransformBase.cc b/src/ufo/variabletransforms/TransformBase.cc index ea049dd8c..613b0bae4 100644 --- a/src/ufo/variabletransforms/TransformBase.cc +++ b/src/ufo/variabletransforms/TransformBase.cc @@ -14,9 +14,9 @@ namespace ufo { TransformBase::TransformBase(const VariableTransformParametersBase& options, const ObsFilterData& data, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) - : options_(options), data_(data) , flags_(flags), obserr_(obserr) { + const std::shared_ptr>& flags, + const std::shared_ptr>& obserr) + : options_(options), data_(data) , flags_(*flags), obserr_(*obserr) { method_ = formulas::resolveMethods(options.Method.value()); UseValidDataOnly_ = options.UseValidDataOnly.value(); obsName_ = data_.obsspace().obsname(); @@ -37,8 +37,8 @@ TransformFactory::TransformFactory(const std::string& name) { std::unique_ptr TransformFactory::create( const std::string& name, const VariableTransformParametersBase& options, const ObsFilterData& data, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) { + const std::shared_ptr>& flags, + const std::shared_ptr>& obserr) { oops::Log::trace() << " --> TransformFactory::create" << std::endl; oops::Log::trace() << " --> name: " << name << std::endl; diff --git a/src/ufo/variabletransforms/TransformBase.h b/src/ufo/variabletransforms/TransformBase.h index 90ffb2847..e00b728e5 100644 --- a/src/ufo/variabletransforms/TransformBase.h +++ b/src/ufo/variabletransforms/TransformBase.h @@ -54,8 +54,8 @@ class TransformBase { public: TransformBase(const VariableTransformParametersBase &options, const ObsFilterData& data, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + const std::shared_ptr>& flags, + const std::shared_ptr>& obserr); /// Destructor virtual ~TransformBase() {} /// Run variable conversion @@ -141,14 +141,15 @@ class TransformBase { // Update QC flags to account for values that were previously missing but // are now present (or vice versa). if (flags_.has(varName)) { - ASSERT(flags_[varName].size() == outputObsVector.size()); + std::vector &varFlags = flags_[varName]; + ASSERT(varFlags.size() == outputObsVector.size()); const T missing = util::missingValue(); for (size_t iloc = 0; iloc < outputObsVector.size(); ++iloc) { - if (flags_[varName][iloc] == QCflags::missing && outputObsVector[iloc] != missing) - flags_[varName][iloc] = QCflags::pass; - else if (flags_[varName][iloc] == QCflags::pass && outputObsVector[iloc] == missing) - flags_[varName][iloc] = QCflags::missing; + if (varFlags[iloc] == QCflags::missing && outputObsVector[iloc] != missing) + varFlags[iloc] = QCflags::pass; + else if (varFlags[iloc] == QCflags::pass && outputObsVector[iloc] == missing) + varFlags[iloc] = QCflags::missing; } } } @@ -169,16 +170,15 @@ class TransformBase { const std::vector & dimList, const std::string &outputTag = "DerivedObsValue") { std::vector outputObsVector(obsVector); - const std::string fullVarName = varName + "_" + channel; // Fill in missing values with values of non-derived group if (options_.FillMissingDerivedFromOriginal && outputTag.substr(0, 7) == "Derived") { std::string originalTag = outputTag; originalTag.erase(0, 7); // Remove Derived from group name - if (obsdb_.has(originalTag, fullVarName)) { + if (obsdb_.has(originalTag, varName + "_" + channel)) { std::vector originalValues(obsdb_.nlocs()); // Need to skipDerived so we pickup original ObsValues - obsdb_.get_db(originalTag, fullVarName, originalValues, {}, true); + obsdb_.get_db(originalTag, varName + "_" + channel, originalValues, {}, true); const T missing = util::missingValue(); for (size_t jloc = 0; jloc < obsdb_.nlocs(); ++jloc) { if (outputObsVector[jloc] == missing && @@ -188,16 +188,17 @@ class TransformBase { } } } - obsdb_.put_db(outputTag, fullVarName, outputObsVector, dimList); - if (flags_.has(fullVarName)) { - ASSERT(flags_[fullVarName].size() == outputObsVector.size()); + obsdb_.put_db(outputTag, varName + "_" + channel, outputObsVector, dimList); + if (flags_.has(varName + "_" + channel)) { + std::vector &varFlags = flags_[varName + "_" + channel]; + ASSERT(varFlags.size() == outputObsVector.size()); const T missing = util::missingValue(); for (size_t iloc = 0; iloc < outputObsVector.size(); ++iloc) { - if (flags_[fullVarName][iloc] == QCflags::missing && outputObsVector[iloc] != missing) - flags_[fullVarName][iloc] = QCflags::pass; - else if (flags_[fullVarName][iloc] == QCflags::pass && outputObsVector[iloc] == missing) - flags_[fullVarName][iloc] = QCflags::missing; + if (varFlags[iloc] == QCflags::missing && outputObsVector[iloc] != missing) + varFlags[iloc] = QCflags::pass; + else if (varFlags[iloc] == QCflags::pass && outputObsVector[iloc] == missing) + varFlags[iloc] = QCflags::missing; } } } @@ -231,8 +232,8 @@ class TransformFactory { static std::unique_ptr create( const std::string &, const VariableTransformParametersBase &, const ObsFilterData &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + const std::shared_ptr> &, + const std::shared_ptr> &); static std::unique_ptr createParameters( const std::string &); virtual ~TransformFactory() = default; @@ -244,8 +245,8 @@ class TransformFactory { virtual std::unique_ptr make( const VariableTransformParametersBase &, const ObsFilterData &, - ioda::ObsDataVector &, - ioda::ObsDataVector &) = 0; + const std::shared_ptr> &, + const std::shared_ptr> &) = 0; virtual std::unique_ptr makeParameters() const = 0; static std::map &getMakers() { static std::map makers_; @@ -266,8 +267,8 @@ class TransformMaker : public TransformFactory { std::unique_ptr make( const VariableTransformParametersBase &options, const ObsFilterData& data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) override { + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) override { const auto &stronglyTypedParams = dynamic_cast(options); return std::unique_ptr(new T(stronglyTypedParams, data, flags, obserr)); } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ca8c69c78..d08963a4b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -146,13 +146,11 @@ list( APPEND ufo_test_input testinput/ufo_geovals_set_invalid_default_format.yaml testinput/ufo_geovals_get_nonexistent_var.yaml testinput/variables.yaml - testinput/unit_tests/obslocalization/obs_localization.yaml ) file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/testinput) file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/testinput/unit_tests) -file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/testinput/unit_tests/obslocalization) CREATE_SYMLINK( ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} ${ufo_test_input} ) # Create Data directory for test output files @@ -364,14 +362,6 @@ ecbuild_add_test( TARGET ufo_sampled_locations ENVIRONMENT OOPS_TRAPFPE=1 LIBS ufo ) -# Test obs-space localization computeLocalization(Point3, Point3) for Box car, GC99, SOAR -ecbuild_add_test( TARGET ufo_obs_localization - SOURCES mains/TestObsLocalization.cc - ufo/ObsLocalization.h - ARGS "testinput/unit_tests/obslocalization/obs_localization.yaml" - ENVIRONMENT OOPS_TRAPFPE=1 - LIBS ufo ) - ecbuild_add_test( TARGET ufo_recordhandler COMMAND ${CMAKE_BINARY_DIR}/bin/test_RecordHandler.x ARGS "testinput/recordhandler.yaml" @@ -702,7 +692,7 @@ if( ${rttov_FOUND} ) set(RTTOV_VERSION_SUFFIX "_rttov12") set(RTTOV_CAMEL_VERSION "2") endif() - + list( APPEND rttov_coef_data rttov7pred54L/rtcoef_dmsp_17_ssmis.dat rttov7pred54L/rtcoef_gcom-w_1_amsr2.dat @@ -732,7 +722,7 @@ if( ${rttov_FOUND} ) pchsr_v9.3.${RTTOV_COEF_EXT} ) - + # Set RTTOV coefficient directory if(DEFINED ENV{RTTOV_LOCAL_COEF_DIR} AND EXISTS $ENV{RTTOV_LOCAL_COEF_DIR}) set(RTTOV_COEF_DIR "$ENV{RTTOV_LOCAL_COEF_DIR}") @@ -751,7 +741,7 @@ if( ${rttov_FOUND} ) set(RTTOV_EMIS_ATLAS_DIR "${rttov_BASE_DIR}/etc/emis_data") endif() message(STATUS "RTTOV emissivity atlas directory: ${RTTOV_EMIS_ATLAS_DIR}") - + # Handle mietable (v12) or hydrotable (v14) if(rttov_VERSION VERSION_GREATER_EQUAL 14.0) # RTTOV v14: use NetCDF hydrotable file directly @@ -780,7 +770,7 @@ if( ${rttov_FOUND} ) list(APPEND rttov_coef_data "mietable/mietable_metop_amsuamhs.bin") endif() endif() - + # Symlink RTcoef files create_symlink_filename( ${RTTOV_COEF_DIR} ${CMAKE_CURRENT_BINARY_DIR}/Data diff --git a/test/mains/TestObsLocalization.cc b/test/mains/TestObsLocalization.cc deleted file mode 100644 index 8540f6af4..000000000 --- a/test/mains/TestObsLocalization.cc +++ /dev/null @@ -1,15 +0,0 @@ -/* - * (C) Copyright 2026 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include "../ufo/ObsLocalization.h" -#include "oops/runs/Run.h" - -int main(int argc, char ** argv) { - oops::Run run(argc, argv); - ufo::test::ObsLocalization tests; - return run.execute(tests); -} diff --git a/test/testinput/instrumentTests/amsua/CMakeLists.txt b/test/testinput/instrumentTests/amsua/CMakeLists.txt index e0b98522a..5dac1c9ef 100644 --- a/test/testinput/instrumentTests/amsua/CMakeLists.txt +++ b/test/testinput/instrumentTests/amsua/CMakeLists.txt @@ -117,6 +117,16 @@ if( crtm_FOUND ) WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../ LIBS ufo ) + # test matching GSI with small tolerance, is this still valid test consider update or remove + ufo_add_test( NAME instrument_amsua_n18_gfs_HofX_bc + ECBUILD + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsOperator.x + ARGS "${CMAKE_CURRENT_SOURCE_DIR}/amsua_n18_gfs_HofX_bc.yaml" + MPI 1 + LABELS crtm amsua n18 bc instrument ci_flake_disable tier2 + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../ + LIBS ufo ) + ufo_add_test( NAME instrument_amsua_n19_gfs_HofX_bc ECBUILD COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsOperator.x diff --git a/test/testinput/instrumentTests/amsua/amsua_n18_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/amsua/amsua_n18_gfs_HofX_bc.yaml new file mode 100644 index 000000000..c71616147 --- /dev/null +++ b/test/testinput/instrumentTests/amsua/amsua_n18_gfs_HofX_bc.yaml @@ -0,0 +1,44 @@ +time window: + begin: 2020-11-01T09:00:00Z + end: 2020-11-01T15:00:00Z + +observations: +- obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_n18 + EndianType: little_endian + CoefficientPath: Data/ + obs space: + name: amsua_n18 + obsdatain: + engine: + type: H5File + obsfile: Data/ufo/testinput_tier_1/instruments/radiance/amsua_n18_obs_2020110112_m.nc4 + simulated variables: [brightnessTemperature] + channels: &all_channels 1-15 + geovals: + filename: Data/ufo/testinput_tier_1/instruments/radiance/amsua_n18_geoval_2020110112_m.nc4 + obs bias: + input file: Data/ufo/testinput_tier_1/instruments/radiance/satbias_amsua_n18.2020110106.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: &amsua_n18_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_n18_tlapmean.txt + - name: lapseRate + tlapse: *amsua_n18_tlap + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + vector ref: GsiHofXBc + tolerance: 1.e-7 diff --git a/test/testinput/unit_tests/errors/CMakeLists.txt b/test/testinput/unit_tests/errors/CMakeLists.txt index 50011de07..4a32d2c5e 100644 --- a/test/testinput/unit_tests/errors/CMakeLists.txt +++ b/test/testinput/unit_tests/errors/CMakeLists.txt @@ -51,7 +51,7 @@ ufo_add_test( NAME obserrordiffusion ARGS "${CMAKE_CURRENT_SOURCE_DIR}/obserrordiffusion.yaml" MPI 1 LIBS ufo - LABELS errors + LABELS errors ci_flake_disable WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../ ) ufo_add_test( NAME obserror_dirac_diffusion @@ -110,13 +110,3 @@ ufo_add_test( NAME reconditioning_mpi LIBS ufo LABELS errors WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../ ) - -ufo_add_test( NAME reconditioning_within_group - TIER 1 - ECBUILD - SOURCES ../../../mains/TestReconditioning.cc - ARGS "${CMAKE_CURRENT_SOURCE_DIR}/reconditioning_withinGroup.yaml" - MPI 1 - LIBS ufo - LABELS errors - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../ ) diff --git a/test/testinput/unit_tests/errors/diracDiffusion.yaml b/test/testinput/unit_tests/errors/diracDiffusion.yaml index f9af66977..6879f7989 100644 --- a/test/testinput/unit_tests/errors/diracDiffusion.yaml +++ b/test/testinput/unit_tests/errors/diracDiffusion.yaml @@ -20,10 +20,6 @@ observations: correlation variable names: [brightnessTemperature] correlation lengthscale: 200000. # meters normalization iterations: 10000 - control grid: - grid spacing in degrees: 10 # control grid resolution in degrees - remove within meters: 200000 # remove control points within this distance from observations (meters) - # use lengthscale plus some padding to impact mesh not diffusion test: reference filename: testref/unit_tests/errors/diracDiffusion.ref diff --git a/test/testinput/unit_tests/errors/obserrorcrossvarcorr.yaml b/test/testinput/unit_tests/errors/obserrorcrossvarcorr.yaml index 58b13ed57..12821cad5 100644 --- a/test/testinput/unit_tests/errors/obserrorcrossvarcorr.yaml +++ b/test/testinput/unit_tests/errors/obserrorcrossvarcorr.yaml @@ -14,7 +14,7 @@ observations: covariance model: cross variable covariances input file: Data/ufo/testinput_tier_1/obserror_corr_3var.nc4 obs error test: - test localization: true + test localization: false - obs space: name: Sondes (cross-var corr, two vars) @@ -27,7 +27,7 @@ observations: covariance model: cross variable covariances input file: Data/ufo/testinput_tier_1/obserror_corr_3var.nc4 obs error test: - test localization: true + test localization: false - obs space: name: Sondes (cross-var corr, three vars) @@ -40,7 +40,7 @@ observations: covariance model: cross variable covariances input file: Data/ufo/testinput_tier_1/obserror_corr_3var.nc4 obs error test: - test localization: true + test localization: false - obs space: name: Sondes (cross-var corr, four vars, one is missing from the corr file) @@ -53,7 +53,7 @@ observations: covariance model: cross variable covariances input file: Data/ufo/testinput_tier_1/obserror_corr_3var.nc4 obs error test: - test localization: true + test localization: false - obs space: name: Sondes (cross-var corr, four vars, one is missing from the corr file) @@ -66,7 +66,7 @@ observations: covariance model: cross variable covariances input file: Data/ufo/testinput_tier_1/obserror_corr_3var.nc4 obs error test: - test localization: true + test localization: false - obs space: name: testReader (cross-var corr with four vars, three vars simulated) @@ -83,7 +83,7 @@ observations: covariance model: cross variable covariances input file: Data/ufo/testinput_tier_1/obserror_corr_test.nc4 obs error test: - test localization: true + test localization: false test reader: true reference: [0.1, 1, 0.15] @@ -102,7 +102,7 @@ observations: covariance model: cross variable covariances input file: Data/ufo/testinput_tier_1/obserror_corr_test.nc4 obs error test: - test localization: true + test localization: false test reader: true reference: [0.2,0.15, 1, 0.7, 0] @@ -121,6 +121,6 @@ observations: covariance model: cross variable covariances input file: Data/ufo/testinput_tier_1/obserror_corr_test.nc4 obs error test: - test localization: true + test localization: false test reader: true reference: [1] diff --git a/test/testinput/unit_tests/errors/obserrorcrossvarcov.yaml b/test/testinput/unit_tests/errors/obserrorcrossvarcov.yaml index ae3eefdf8..bd518758e 100644 --- a/test/testinput/unit_tests/errors/obserrorcrossvarcov.yaml +++ b/test/testinput/unit_tests/errors/obserrorcrossvarcov.yaml @@ -1,6 +1,6 @@ time window: - begin: 2020-11-01T06:00:00Z - end: 2020-11-01T18:00:00Z + begin: 2018-04-14T12:00:00Z + end: 2018-04-15T12:00:00Z observations: @@ -67,9 +67,8 @@ observations: obs error: covariance model: cross variable covariances input file: Data/ufo/testinput_tier_1/Rcov_iasi_metop-b_sea.nc4 - Obs Error test tolerance: 1e-9 obs error test: - test localization: true + test localization: false - obs space: name: iasi_metop-b (channels == all channels in the correlations file) @@ -98,6 +97,5 @@ observations: obs error: covariance model: cross variable covariances input file: Data/ufo/testinput_tier_1/Rcov_iasi_metop-b_sea.nc4 - Obs Error test tolerance: 1e-9 obs error test: - test localization: true + test localization: false diff --git a/test/testinput/unit_tests/errors/obserrordiffusion.yaml b/test/testinput/unit_tests/errors/obserrordiffusion.yaml index e0b519d68..727879130 100644 --- a/test/testinput/unit_tests/errors/obserrordiffusion.yaml +++ b/test/testinput/unit_tests/errors/obserrordiffusion.yaml @@ -17,9 +17,5 @@ observations: correlation lengthscale: 5000. normalization iterations: 1000 Obs Error test tolerance: 1.e-8 - control grid: - grid spacing in degrees: 10 # control grid resolution in degrees - remove within meters: 200000 # remove control points within this distance from observations (meters) - # use lengthscale plus some padding to impact mesh not diffusion obs error test: test localization: false diff --git a/test/testinput/unit_tests/errors/obserrorwithingroupcorr.yaml b/test/testinput/unit_tests/errors/obserrorwithingroupcorr.yaml index 8a978c690..93434edf5 100644 --- a/test/testinput/unit_tests/errors/obserrorwithingroupcorr.yaml +++ b/test/testinput/unit_tests/errors/obserrorwithingroupcorr.yaml @@ -59,23 +59,3 @@ observations: correlation lengthscale: 13000. # in m obs error test: test localization: false - -- obs space: - name: Sondes (within group covariances with gaussian correlation function and basic reconditioning) - obsdatain: - engine: - type: H5File - obsfile: Data/ufo/testinput_tier_1/sondes_obs_2018041500_s.nc4 - obsgrouping: - group variables: [sequenceNumber] - sort variable: pressure - sort order: ascending - simulated variables: [airTemperature, windEastward, windNorthward] - obs error: - covariance model: within group covariances - distance function: haversine - correlation function: gaussian - apply basic reconditioning: true - correlation lengthscale: 1400. # in m - obs error test: - test localization: false diff --git a/test/testinput/unit_tests/errors/reconditioning_withinGroup.yaml b/test/testinput/unit_tests/errors/reconditioning_withinGroup.yaml deleted file mode 100644 index b22878fc1..000000000 --- a/test/testinput/unit_tests/errors/reconditioning_withinGroup.yaml +++ /dev/null @@ -1,27 +0,0 @@ -time window: - begin: 2018-04-14T21:00:00Z - end: 2018-04-15T03:00:00Z - -observations: -- obs space: - name: Sondes (within group covariances using haversine distance and gaussian correlation function w/ advanced reconditioner) - obsdatain: - engine: - type: H5File - obsfile: Data/ufo/testinput_tier_1/sondes_obs_2018041500_s.nc4 - obsgrouping: - group variables: [sequenceNumber] - sort variable: pressure - sort order: ascending - simulated variables: [airTemperature, windEastward, windNorthward] - obs error: - covariance model: within group covariances - distance function: haversine - correlation function: gaussian - correlation lengthscale: 2000. # in m - reconditioning: - recondition method: Ridge Regression - fraction: 0.9 - obs error test: - test reader: true - reference: [322.344,391.1,459.624,444.458,424.292,669.169,627.638,229.982,230.075,213.214,272.439,212.957,369.691,369.511,49.9851,2.77498,71.3079,-29.7157,254.176,71.4432,601.438,-61.1725,629.321,-66.1769,571.344,49.2934,230.431,72.1513,873.28,75.4791,831.886,71.9031,452.113,-62.8127,119.552,263.853,119.597,266.688,119.389,268.971,112.644,262.727,245.007,74.1117,266.9,78.3315,334.494,88.8221,306.8,85.1776,550.066,-33.0129,364.036,90.9901,594.635,-35.2926,408.446,17.6729,387.861,91.463,475.366,-3.80788,473.885,-2.95747,451.963,-20.5918,74.6161,269.549,410.318,-16.7671,449.217,-10.0663,343.009,5.49996,432.602,-23.5831,64.7918,261.927,518.162,68.7194,491.718,-1.27451,508.145,0.949498,533.051,61.7235,562.933,27.8447,561.976,38.6925,40.6551,274.177,32.1251,264.328,24.5098,267.456,20.9273,269.011,8.72175,273.495,3.45904,274.946,-47.1277,272.72,-55.289,273.7,-152.623,253.867,-161.42,252.169,-183.645,230.035,-177.844,215.183,-185.375,211,-176.831,196.776,-179.829,193.691,-180.072,193.4,-189.709,168.838,-189.879,166.643,-190.467,157.146,-179.252,130.908,-173.781,120.216,-175.561,122.31,-165.76,112.216,-160.996,107.915,-155.535,103.262,-150.172,99.0564,-138.639,90.9216,-137.471,90.1345] diff --git a/test/testinput/unit_tests/filters/obsfunctions/CMakeLists.txt b/test/testinput/unit_tests/filters/obsfunctions/CMakeLists.txt index e29d9770b..5dd4a69cf 100644 --- a/test/testinput/unit_tests/filters/obsfunctions/CMakeLists.txt +++ b/test/testinput/unit_tests/filters/obsfunctions/CMakeLists.txt @@ -53,15 +53,6 @@ ufo_add_test( NAME function_arithmetic LABELS filters obsfunctions WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../../ ) # -ufo_add_test( NAME function_circulardifference - ECBUILD - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "${CMAKE_CURRENT_SOURCE_DIR}/function_circulardifference.yaml" - MPI 1 - LIBS ufo - LABELS filters obsfunctions - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../../ ) -# ufo_add_test( NAME function_aircraft_bc_offline ECBUILD COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x @@ -846,7 +837,7 @@ ufo_add_test( NAME function_solarzenith_skiprejected LIBS ufo LABELS filters obsfunctions WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../../ ) - + # Test for the string manipulation obsfucntion ufo_add_test( NAME function_stringmanipulation ECBUILD @@ -855,8 +846,8 @@ ufo_add_test( NAME function_stringmanipulation MPI 1 LIBS ufo LABELS filters obsfunctions - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../../ ) - + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../../ ) + # ufo_add_test( NAME function_surfacecloudmodellevelcbh ECBUILD @@ -884,24 +875,7 @@ ufo_add_test( NAME function_geocloudcreatecloudcolumn LIBS ufo LABELS filters obsfunctions WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../../ ) -# -ufo_add_test( NAME function_timebinner - ECBUILD - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "${CMAKE_CURRENT_SOURCE_DIR}/function_timebinner.yaml" - LIBS ufo - LABELS filters obsfunctions - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../../ ) -# -ufo_add_test( NAME function_timebinner_mpi - ECBUILD - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "${CMAKE_CURRENT_SOURCE_DIR}/function_timebinner.yaml" - MPI 3 - LIBS ufo - LABELS filters obsfunctions - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../../ ) -# + ufo_add_test( NAME function_tropo_estimate ECBUILD COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x diff --git a/test/testinput/unit_tests/filters/obsfunctions/function_circulardifference.yaml b/test/testinput/unit_tests/filters/obsfunctions/function_circulardifference.yaml deleted file mode 100644 index 0ce11fa88..000000000 --- a/test/testinput/unit_tests/filters/obsfunctions/function_circulardifference.yaml +++ /dev/null @@ -1,47 +0,0 @@ -time window: - begin: 2000-01-01T00:00:00Z - end: 2030-01-01T00:00:00Z - -observations: - - obs space: - name: Test ObsFunction CircularDifference - obsdatain: - engine: - type: H5File - obsfile: Data/ufo/testinput_tier_1/circular_difference_test_obs.nc4 - simulated variables: [] - obs filters: - # signed between windDirectionA and windDirectionB (windDirectionB - windDirectionA) - - filter: Variable Assignment - assignments: - - name: MetaData/signedWindDirectionCircularDiff - type: float - function: - name: ObsFunction/CircularDifference - options: - variable start: ObsValue/windDirectionA - variable end: ObsValue/windDirectionB - signed: true - circular period: 360.0 - # unsigned between windDirectionA and windDirectionB (absolute value of windDirectionB - windDirectionA) - - filter: Variable Assignment - assignments: - - name: MetaData/unsignedWindDirectionCircularDiff - type: float - function: - name: ObsFunction/CircularDifference - options: - variable start: ObsValue/windDirectionA - variable end: ObsValue/windDirectionB - signed: false - circular period: 360.0 - compareVariables: - - test: - name: MetaData/signedWindDirectionCircularDiff - reference: - name: TestReference/signedWindDirectionCircularDiff - relTol: 1e-6 - - test: - name: MetaData/unsignedWindDirectionCircularDiff - reference: - name: TestReference/unsignedWindDirectionCircularDiff diff --git a/test/testinput/unit_tests/filters/obsfunctions/function_timebinner.yaml b/test/testinput/unit_tests/filters/obsfunctions/function_timebinner.yaml deleted file mode 100644 index d22b832b5..000000000 --- a/test/testinput/unit_tests/filters/obsfunctions/function_timebinner.yaml +++ /dev/null @@ -1,208 +0,0 @@ -time window: - begin: 2000-01-01T00:00:00Z - end: 2030-01-01T00:00:00Z - -observations: - - obs space: - name: Test ObsFunction TimeBinner - obsdatain: - engine: - type: H5File - obsfile: Data/ufo/testinput_tier_1/time_binner_test_obs.nc4 - simulated variables: [] - obs filters: - # Note: bin centres are floored from the first earliest timestamp - # globally, so, for 2018-04-17 01:20:02 UTC the first minutely bin is - # centred on 01:20:00 UTC, the first houly bin is centred on 01:00:00 UTC, - # the first daily bin on 00:00:00 UTC, etc. - # If "reverse chronological order: true" then bins are derived from the - # latest timestamp instead. - - # (A) T-30M <= t <= T where T is the top of the hour. dateTimes outside of - # this range get missing bin numbers. "binned timestamp variable" should - # take on the hour value T for all dateTimes in the bin, others get missing. - - filter: Variable Assignment - assignments: - - name: MetaData/aboutHourMinus30MBinNumbers - type: int - function: - name: IntObsFunction/TimeBinner - options: - bin interval unit: hour - bin window lower bound: -PT30M # 30 minutes before the hour (inclusive) - bin window upper bound: PT0M # up to the hour (inclusive) - input timestamp variable: MetaData/dateTime # default - binned timestamp variable: MetaData/aboutHourMinus30MTimestamps # optional - all values in a bin get the same timestamp - timestamps outside of bin are set to missing - # (B) As above, but for T-30M <= t <= T+PT29M59S (no missing values - # expected since OOPS dateTimes are to the nearest second). - - filter: Variable Assignment - assignments: - - name: MetaData/aboutHourMinus30MPlus29M59SBinNumbers - type: int - function: - name: IntObsFunction/TimeBinner - options: - bin interval unit: hour - bin window lower bound: -PT30M # 30 minutes before the hour (inclusive) - bin window upper bound: PT29M59S # up to 29 minutes 59 seconds after the hour (inclusive) - input timestamp variable: MetaData/dateTime # default - binned timestamp variable: MetaData/aboutHourMinus30MPlus29M59STimestamps - # (C) T < t <= T+PT1H (i.e. the hour after the top of the hour) - use that - # OOPS times are to the nearest second to avoid ambiguity at the hour. - - filter: Variable Assignment - assignments: - - name: MetaData/excludingHourPlus1HourBinNumbers - type: int - function: - name: IntObsFunction/TimeBinner - options: - bin interval unit: hour - bin window lower bound: PT1S # from the hour (exclusive) or 1S after the hour (inclusive) - bin window upper bound: PT1H # up to one hour after the hour (inclusive) - input timestamp variable: MetaData/dateTime # default - binned timestamp variable: MetaData/excludingHourPlus1HourBinTimestamps - # (D) Test day interval, i.e. T-12H <= t <= T where T is midnight UTC. - - filter: Variable Assignment - assignments: - - name: MetaData/aboutMidnightMinus12HBinNumbers - type: int - function: - name: IntObsFunction/TimeBinner - options: - bin interval unit: day - bin window lower bound: -PT12H # 12 hours before midnight (inclusive) - bin window upper bound: PT0M # up to midnight (inclusive) - input timestamp variable: MetaData/dateTime # default - binned timestamp variable: MetaData/aboutMidnightMinus12HTimestamps - # (E) Test minute interval: T-30S <= t <= T+10S where T is the minute. - - filter: Variable Assignment - assignments: - - name: MetaData/aboutMinuteMinus30SPlus10SBinNumbers - type: int - function: - name: IntObsFunction/TimeBinner - options: - bin interval unit: minute - bin window lower bound: -PT30S # 30 seconds before the minute (inclusive) - bin window upper bound: PT10S # up to 10 seconds after the minute (inclusive) - input timestamp variable: MetaData/dateTime # default - binned timestamp variable: MetaData/aboutMinuteMinus30SPlus10STimestamps - # (F) Test reverse chronological order: T-30M <= t <= T+PT29M59S where T - # is the hour, but deriving bins from the latest timestamp back to earliest. - - filter: Variable Assignment - assignments: - - name: MetaData/reverseAboutHourMinus30MPlus29M59SBinNumbers - type: int - function: - name: IntObsFunction/TimeBinner - options: - bin interval unit: hour - bin window lower bound: -PT30M # 30 minutes before the hour (inclusive) - bin window upper bound: PT29M59S # up to 29 minutes 59 seconds after the hour (inclusive) - input timestamp variable: MetaData/dateTime # default - binned timestamp variable: MetaData/reverseAboutHourMinus30MPlus29M59STimestamps - reverse chronological order: true # default is false - # (G) Test reverse chronological order: T-30M <= t <= T where T is the hour. - # deriving bins from the latest timestamp back to earliest. - - filter: Variable Assignment - assignments: - - name: MetaData/reverseAboutHourMinus30MBinNumbers - type: int - function: - name: IntObsFunction/TimeBinner - options: - bin interval unit: hour - bin window lower bound: -PT30M # 30 minutes before the hour (inclusive) - bin window upper bound: PT0M # up to the hour (inclusive) - input timestamp variable: MetaData/dateTime # default - binned timestamp variable: MetaData/reverseAboutHourMinus30MTimestamps - reverse chronological order: true # default is false - # (H) Test reverse chronological order with opposite offset to (G): - # T <= t <= T+30M where T is the hour, deriving bins from the latest timestamp - # back to earliest. - - filter: Variable Assignment - assignments: - - name: MetaData/reverseAboutHourPlus30MBinNumbers - type: int - function: - name: IntObsFunction/TimeBinner - options: - bin interval unit: hour - bin window lower bound: PT0M # from the hour (inclusive) - bin window upper bound: PT30M # up to 30 minutes after the hour (inclusive) - input timestamp variable: MetaData/dateTime # default - binned timestamp variable: MetaData/reverseAboutHourPlus30MTimestamps - reverse chronological order: true # default is false - compareVariables: - # (A) - - test: - name: MetaData/aboutHourMinus30MBinNumbers - reference: - name: TestReference/aboutHourMinus30MBinNumbers - - test: - name: MetaData/aboutHourMinus30MTimestamps - reference: - name: TestReference/aboutHourMinus30MTimestamps - # (B) - - test: - name: MetaData/aboutHourMinus30MPlus29M59SBinNumbers - reference: - name: TestReference/aboutHourMinus30MPlus29M59SBinNumbers - - test: - name: MetaData/aboutHourMinus30MPlus29M59STimestamps - reference: - name: TestReference/aboutHourMinus30MPlus29M59STimestamps - # (C) - - test: - name: MetaData/excludingHourPlus1HourBinNumbers - reference: - name: TestReference/excludingHourPlus1HourBinNumbers - - test: - name: MetaData/excludingHourPlus1HourBinTimestamps - reference: - name: TestReference/excludingHourPlus1HourBinTimestamps - # (D) - - test: - name: MetaData/aboutMidnightMinus12HBinNumbers - reference: - name: TestReference/aboutMidnightMinus12HBinNumbers - - test: - name: MetaData/aboutMidnightMinus12HTimestamps - reference: - name: TestReference/aboutMidnightMinus12HTimestamps - # (E) - - test: - name: MetaData/aboutMinuteMinus30SPlus10SBinNumbers - reference: - name: TestReference/aboutMinuteMinus30SPlus10SBinNumbers - - test: - name: MetaData/aboutMinuteMinus30SPlus10STimestamps - reference: - name: TestReference/aboutMinuteMinus30SPlus10STimestamps - # (F) - - test: - name: MetaData/reverseAboutHourMinus30MPlus29M59SBinNumbers - reference: - name: TestReference/reverseAboutHourMinus30MPlus29M59SBinNumbers - - test: - name: MetaData/reverseAboutHourMinus30MPlus29M59STimestamps - reference: - name: TestReference/aboutHourMinus30MPlus29M59STimestamps # should be the same as (B) - # (G) - - test: - name: MetaData/reverseAboutHourMinus30MBinNumbers - reference: - name: TestReference/reverseAboutHourMinus30MBinNumbers - - test: - name: MetaData/reverseAboutHourMinus30MTimestamps - reference: - name: TestReference/aboutHourMinus30MTimestamps # should be the same as (A) - # (H) - - test: - name: MetaData/reverseAboutHourPlus30MBinNumbers - reference: - name: TestReference/reverseAboutHourPlus30MBinNumbers - - test: - name: MetaData/reverseAboutHourPlus30MTimestamps - reference: - name: TestReference/reverseAboutHourPlus30MTimestamps # would be the same as non-reverse diff --git a/test/testinput/unit_tests/obslocalization/obs_localization.yaml b/test/testinput/unit_tests/obslocalization/obs_localization.yaml deleted file mode 100644 index 33208b8d8..000000000 --- a/test/testinput/unit_tests/obslocalization/obs_localization.yaml +++ /dev/null @@ -1,78 +0,0 @@ -time window: - begin: 2000-01-01T00:00:00Z - end: 2000-01-01T06:00:00Z - -obs space: - name: simple obs - simulated variables: [windEastward] - obsdatain: - engine: - type: GenList - lons: [10.0, 20.0, 30.0] - lats: [5.0, 15.0, 25.0] - dateTimes: [3600, 3600, 3600] - epoch: "seconds since 2000-01-01T00:00:00Z" - obs errors: [1.0] - -obs localizations: - # Box car: returns 1.0 if distance <= lengthscale, 0.0 otherwise -- localization: - localization method: Horizontal Box car - lengthscale: 1000000.0 # 1000 km - point pair tests: - # Same point -> distance = 0 -> returns 1.0 - - p1: [0.0, 0.0, 0.0] - p2: [0.0, 0.0, 0.0] - reference value: 1.0 - tolerance: 1.0e-10 - # Points ~1111 km apart (10 deg longitude at equator) -> outside 1000 km -> 0.0 - - p1: [0.0, 0.0, 0.0] - p2: [10.0, 0.0, 0.0] - reference value: 0.0 - tolerance: 1.0e-10 - # Points ~556 km apart (5 deg longitude at equator) -> inside 1000 km -> 1.0 - - p1: [0.0, 0.0, 0.0] - p2: [5.0, 0.0, 0.0] - reference value: 1.0 - tolerance: 1.0e-10 - - # GC99: applies Gaspari-Cohn taper; cutoff at lengthscale (r > 1 -> 0) -- localization: - localization method: Horizontal Gaspari-Cohn - lengthscale: 1000000.0 # 1000 km - point pair tests: - # Same point -> distance = 0 -> gc99(0) = 1.0 - - p1: [0.0, 0.0, 0.0] - p2: [0.0, 0.0, 0.0] - reference value: 1.0 - tolerance: 1.0e-10 - # Points ~1111 km apart -> outside 1000 km cutoff -> 0.0 - - p1: [0.0, 0.0, 0.0] - p2: [10.0, 0.0, 0.0] - reference value: 0.0 - tolerance: 1.0e-10 - - # SOAR: applies (1 + x)*exp(-x) taper; cutoff at lengthscale -- localization: - localization method: Horizontal SOAR - lengthscale: 1000000.0 # 1000 km - soar horizontal decay: 1.0e-6 # 1/m -> x = distance * decay - point pair tests: - # Same point -> distance = 0 -> soar(0) = (1+0)*exp(0) = 1.0 - - p1: [0.0, 0.0, 0.0] - p2: [0.0, 0.0, 0.0] - reference value: 1.0 - tolerance: 1.0e-10 - # Points ~1111 km apart -> outside 1000 km cutoff -> 0.0 - - p1: [0.0, 0.0, 0.0] - p2: [10.0, 0.0, 0.0] - reference value: 0.0 - tolerance: 1.0e-10 - # Points ~556 km apart (5 deg longitude at equator) -> inside cutoff - # distance ~ 555995 m (using atlas Earth radius 6371229 m) - # x = 555995 * 1e-6 = 0.555995 - # soar(0.555995) = (1 + 0.555995) * exp(-0.555995) ~= 0.89237 - - p1: [0.0, 0.0, 0.0] - p2: [5.0, 0.0, 0.0] - reference value: 0.89237 - tolerance: 1.0e-3 diff --git a/test/testinput/unit_tests/predictors/CMakeLists.txt b/test/testinput/unit_tests/predictors/CMakeLists.txt index 961871840..d81c94e6b 100644 --- a/test/testinput/unit_tests/predictors/CMakeLists.txt +++ b/test/testinput/unit_tests/predictors/CMakeLists.txt @@ -135,15 +135,6 @@ ufo_add_test( NAME bias_linear_op LABELS unit_tests predictors WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../ ) # -ufo_add_test( NAME bias_linear_op_tlad - ECBUILD - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsOperatorTLAD.x - ARGS "${CMAKE_CURRENT_SOURCE_DIR}/bias_linear_op_tlad.yaml" - MPI 1 - LIBS ufo - LABELS unit_tests predictors - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../ ) -# # Test for application of bias prediction by record ufo_add_test( NAME aircraft_bc ECBUILD diff --git a/test/testinput/unit_tests/predictors/bias_linear_op_tlad.yaml b/test/testinput/unit_tests/predictors/bias_linear_op_tlad.yaml deleted file mode 100644 index 1f1a0f9f4..000000000 --- a/test/testinput/unit_tests/predictors/bias_linear_op_tlad.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# Test obs bias increment operator - -time window: - begin: 2021-07-31T20:59:00Z - end: 2021-08-01T03:00:00Z - -observations: -- obs operator: - name: VertInterp - vertical coordinate: air_pressure - observation vertical coordinate: pressure - interpolation method: log-linear - obs space: - name: aircraft - obsdatain: - obsgrouping: - group variables: ["stationIdentification"] - sort variable: "pressure" - sort order: "descending" - engine: - type: H5File - obsfile: Data/ufo/testinput_tier_1/aircraftBCOffline_obs.nc4 - simulated variables: [airTemperature] - geovals: - filename: Data/ufo/testinput_tier_1/aircraftBCOffline_geoval.nc4 - obs bias: - input file: Data/ufo/testinput_tier_1/aircraft_bias_20210731T18Z.nc4 - bc by record: true - variational bc: - predictors: - - name: constant - - name: obsMetadataPredictor - variable: aircraftAscentRate - - name: obsMetadataPredictor - variable: aircraftAscentRate - order: 2 - covariance: # TODO: all of this section needs revisiting by experts, this is copy-paste from amsua yaml - minimal required obs number: 20 - variance range: [1.0e-6, 10.0] - step size: 1.0e-4 - largest analysis variance: 10000.0 - prior: - input file: Data/ufo/testinput_tier_1/aircraft_bias_20210731T18Z.nc4 - inflation: - ratio: 1.1 - ratio for small dataset: 2.0 - linear obs operator test: - coef TL: 1.e-5 - tolerance TL: 1.0e-10 - tolerance AD: 1.0e-13 diff --git a/test/testref/unit_tests/errors/diracDiffusion.ref b/test/testref/unit_tests/errors/diracDiffusion.ref index d53c3528f..3c1f84e67 100644 --- a/test/testref/unit_tests/errors/diracDiffusion.ref +++ b/test/testref/unit_tests/errors/diracDiffusion.ref @@ -1,6 +1,6 @@ Diffusion Dirac R*dy stats: -abi_g16 nobs= 100 Min=0.0000000000000000e+00, Max=5.8802631242318482e+00, RMS=5.9146511868699336e-01 +abi_g16 nobs= 100 Min=0.0000000000000000e+00, Max=5.7726513965964150e+00, RMS=5.8243139711273795e-01 -Final peak of dirac impulse: 5.8802631242318482e+00 -Propogated covariance at location 38: 3.0092282517488306e-01 -Propogated covariance at location 51: 5.2443889635492080e-01 +Final peak of dirac impulse: 5.7726513965964150e+00 +Propogated covariance at location 38: 4.2550394030068034e-01 +Propogated covariance at location 51: 5.7314047356895192e-01 diff --git a/test/ufo/ConventionalProfileProcessing.h b/test/ufo/ConventionalProfileProcessing.h index cbbcc3549..8bfbbf78a 100644 --- a/test/ufo/ConventionalProfileProcessing.h +++ b/test/ufo/ConventionalProfileProcessing.h @@ -74,9 +74,11 @@ void testConventionalProfileProcessing(const eckit::LocalConfiguration &conf) { const Variables diagvars(varconfs); const ObsDiagnostics obsdiags(obsdiagconf, obsspace, diagvars.toOopsObsVariables()); - ioda::ObsDataVector obserr(obsspace, obsspace.obsvariables(), "ObsError"); + std::shared_ptr> obserr(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables(), "ObsError")); - ioda::ObsDataVector qcflags(obsspace, obsspace.obsvariables()); + std::shared_ptr> qcflags(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables())); const eckit::LocalConfiguration filterConf(conf, "Conventional Profile Processing"); ufo::ConventionalProfileProcessingParameters filterParameters; @@ -159,7 +161,7 @@ void testConventionalProfileProcessing(const eckit::LocalConfiguration &conf) { std::vector apply(obsspace.nlocs(), true); std::vector> flagged; ProfileDataHandler profileDataHandler(filterdata, - qcflags, + *qcflags, options.DHParameters, apply, filtervars, @@ -182,7 +184,7 @@ void testConventionalProfileProcessing(const eckit::LocalConfiguration &conf) { std::vector apply(obsspace.nlocs(), true); std::vector> flagged; ProfileDataHandler profileDataHandler(filterdata, - qcflags, + *qcflags, options.DHParameters, apply, filtervars, @@ -248,7 +250,7 @@ void testConventionalProfileProcessing(const eckit::LocalConfiguration &conf) { std::vector apply(obsspace.nlocs(), true); std::vector> flagged; ProfileDataHandler profileDataHandler(filterdata, - qcflags, + *qcflags, options.DHParameters, apply, filtervars, @@ -346,7 +348,7 @@ void testConventionalProfileProcessing(const eckit::LocalConfiguration &conf) { std::vector apply(obsspace.nlocs(), true); std::vector> flagged; ProfileDataHandler profileDataHandler(filterdata, - qcflags, + *qcflags, options.DHParameters, apply, filtervars, diff --git a/test/ufo/GaussianThinning.h b/test/ufo/GaussianThinning.h index 6b58cf9f4..6a8a895a7 100644 --- a/test/ufo/GaussianThinning.h +++ b/test/ufo/GaussianThinning.h @@ -62,8 +62,10 @@ void testGaussianThinning(const eckit::LocalConfiguration &conf) { obsspace.put_db("MetaData", "priority", priorities); } - ioda::ObsDataVector obserr(obsspace, obsspace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags(obsspace, obsspace.obsvariables()); + std::shared_ptr> obserr(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables(), "ObsError")); + std::shared_ptr> qcflags(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables())); eckit::LocalConfiguration filterConf(conf, "GaussianThinning"); ufo::GaussianThinningParameters filterParameters; @@ -86,8 +88,8 @@ void testGaussianThinning(const eckit::LocalConfiguration &conf) { const std::vector expectedThinnedObsIndices = conf.getUnsignedVector("expected_thinned_obs_indices"); std::vector thinnedObsIndices; - for (size_t i = 0; i < qcflags.nlocs(); ++i) - if (qcflags[0][i] == ufo::QCflags::thinned) + for (size_t i = 0; i < qcflags->nlocs(); ++i) + if ((*qcflags)[0][i] == ufo::QCflags::thinned) thinnedObsIndices.push_back(i); EXPECT_EQUAL(thinnedObsIndices.size(), expectedThinnedObsIndices.size()); const bool equal = std::equal(thinnedObsIndices.begin(), thinnedObsIndices.end(), diff --git a/test/ufo/GeoVaLs.h b/test/ufo/GeoVaLs.h index c926ac854..520337bce 100644 --- a/test/ufo/GeoVaLs.h +++ b/test/ufo/GeoVaLs.h @@ -103,7 +103,7 @@ void testGeoVaLs() { gv.reorderzdir("air_pressure_levels", flipto); std::vector gvar(nobs); std::vector gvarref(nobs); - float sum = 0.0f; + float sum; for (size_t i = 0; i < ingeovars.size(); ++i) { size_t nlevs = gv.nlevs(ingeovars[i]); sum = 0; diff --git a/test/ufo/HistoryCheck.h b/test/ufo/HistoryCheck.h index 2971cc2d8..fa1c0526d 100644 --- a/test/ufo/HistoryCheck.h +++ b/test/ufo/HistoryCheck.h @@ -51,8 +51,10 @@ void testHistoryCheck(const eckit::LocalConfiguration &conf) { obsspace.put_db("MetaData", "stationIdentification", stationIds); } - ioda::ObsDataVector obserr(obsspace, obsspace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags(obsspace, obsspace.obsvariables()); + std::shared_ptr> obserr(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables(), "ObsError")); + std::shared_ptr> qcflags(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables())); ufo::HistoryCheck filter(obsspace, filterParameters, qcflags, obserr, conf); filter.preProcess(); @@ -60,8 +62,8 @@ void testHistoryCheck(const eckit::LocalConfiguration &conf) { const std::vector expectedRejectedObsIndices = conf.getUnsignedVector("expected rejected obs indices"); std::vector rejectedObsIndices; - for (size_t i = 0; i < qcflags.nlocs(); ++i) - if (qcflags[0][i] == ufo::QCflags::history) + for (size_t i = 0; i < qcflags->nlocs(); ++i) + if ((*qcflags)[0][i] == ufo::QCflags::history) rejectedObsIndices.push_back(i); EXPECT_EQUAL(rejectedObsIndices, expectedRejectedObsIndices); } diff --git a/test/ufo/MetOfficeBuddyCheck.h b/test/ufo/MetOfficeBuddyCheck.h index f253aeca2..f3b490024 100644 --- a/test/ufo/MetOfficeBuddyCheck.h +++ b/test/ufo/MetOfficeBuddyCheck.h @@ -58,8 +58,10 @@ void testMetOfficeBuddyCheck(const eckit::LocalConfiguration &conf) { obsSpace.put_db(varGroup, varName, values); } - ioda::ObsDataVector obserr(obsSpace, obsSpace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags(obsSpace, obsSpace.obsvariables()); + std::shared_ptr> obserr(new ioda::ObsDataVector( + obsSpace, obsSpace.obsvariables(), "ObsError")); + std::shared_ptr> qcflags(new ioda::ObsDataVector( + obsSpace, obsSpace.obsvariables())); eckit::LocalConfiguration filterConf(conf, "Met Office Buddy Check"); ufo::MetOfficeBuddyCheckParameters filterParameters; diff --git a/test/ufo/MetOfficeBuddyPairFinder.h b/test/ufo/MetOfficeBuddyPairFinder.h index a1de85373..f0e846301 100644 --- a/test/ufo/MetOfficeBuddyPairFinder.h +++ b/test/ufo/MetOfficeBuddyPairFinder.h @@ -188,6 +188,8 @@ std::vector findBuddyPairs(const MetOfficeBuddyCheckParamete } void testInvarianceToLongitude(const eckit::LocalConfiguration &conf) { + const float searchRadius = 100; // km + std::vector referenceLatitudes = conf.getFloatVector("reference.latitudes"); std::vector referenceLongitudes = conf.getFloatVector("reference.longitudes"); diff --git a/test/ufo/ObsErrorAssign.h b/test/ufo/ObsErrorAssign.h index 089f1687e..f8dbc0d67 100644 --- a/test/ufo/ObsErrorAssign.h +++ b/test/ufo/ObsErrorAssign.h @@ -49,8 +49,10 @@ void testObsErrorAssign(const eckit::LocalConfiguration &conf) { obsspace.put_db("MetaData", "stringVar", dat); } - ioda::ObsDataVector obserr(obsspace, obsspace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags(obsspace, obsspace.obsvariables()); + std::shared_ptr> obserr(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables(), "ObsError")); + std::shared_ptr> qcflags(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables())); const eckit::LocalConfiguration filterConf(conf, "ObsError assign"); ufo::BlackListParameters bfparam; @@ -75,9 +77,9 @@ void testObsErrorAssign(const eckit::LocalConfiguration &conf) { } } int ind = 0; - for (size_t varn = 0; varn < obserr.nvars(); ++varn) { - for (size_t locn = 0; locn < obserr.nlocs(); ++locn) { - EXPECT(oops::is_close_absolute(obserr[varn][locn], expectedObsError[ind], 1e-4f, 0, + for (size_t varn = 0; varn < obserr->nvars(); ++varn) { + for (size_t locn = 0; locn < obserr->nlocs(); ++locn) { + EXPECT(oops::is_close_absolute((*obserr)[varn][locn], expectedObsError[ind], 1e-4f, 0, oops::TestVerbosity::LOG_SUCCESS_AND_FAILURE)); ind++; } diff --git a/test/ufo/ObsFilters.h b/test/ufo/ObsFilters.h index c219b139e..ce0a101e1 100644 --- a/test/ufo/ObsFilters.h +++ b/test/ufo/ObsFilters.h @@ -191,10 +191,10 @@ class ObsFiltersParameters : public oops::Parameters { //! is not available. //! void runFinalCheck(ioda::ObsSpace &obsspace, - ioda::ObsDataVector &qcflags, - ioda::ObsDataVector &obserr) { + std::shared_ptr> qc_flags, + std::shared_ptr> obserr) { FinalCheck finalCheck(obsspace, FinalCheckParameters(), - qcflags, + qc_flags, obserr); finalCheck.doFilter(); } @@ -222,7 +222,7 @@ void convertLocalObsIndicesToGlobal(std::vector &indices, //! Return the indices of observations whose quality control flags satisfy the //! predicate in at least one variable. //! -//! \param qcflags +//! \param qc_flags //! Vector of quality control flags for all observations. //! \param obsDisttribution //! The MPI distribution used by the ObsSpace. @@ -233,16 +233,16 @@ void convertLocalObsIndicesToGlobal(std::vector &indices, //! template std::vector getObservationIndicesWhere( - const ioda::ObsDataVector &qcflags, + const ioda::ObsDataVector &qc_flags, const eckit::mpi::Comm &comm, const std::vector &globalIdxFromLocalIdx, const Predicate &predicate) { // Among the locations held on the calling process, identify those that satisfy the predicate. std::vector indices; - for (size_t locIndex = 0; locIndex < qcflags.nlocs(); ++locIndex) { + for (size_t locIndex = 0; locIndex < qc_flags.nlocs(); ++locIndex) { bool satisfied = false; - for (size_t varIndex = 0; varIndex < qcflags.nvars(); ++varIndex) { - if (predicate(qcflags[varIndex][locIndex])) { + for (size_t varIndex = 0; varIndex < qc_flags.nvars(); ++varIndex) { + if (predicate(qc_flags[varIndex][locIndex])) { satisfied = true; break; } @@ -268,10 +268,10 @@ std::vector getObservationIndicesWhere( //! Return the indices of observations that have passed quality control in //! at least one variable. //! -std::vector getPassedObservationIndices(const ioda::ObsDataVector & qcflags, +std::vector getPassedObservationIndices(const ioda::ObsDataVector & qc_flags, const eckit::mpi::Comm &comm, const std::vector &globalIdxFromLocalIdx) { - return getObservationIndicesWhere(qcflags, comm, globalIdxFromLocalIdx, + return getObservationIndicesWhere(qc_flags, comm, globalIdxFromLocalIdx, [](int qcFlag) { return qcFlag == 0; }); } @@ -281,10 +281,10 @@ std::vector getPassedObservationIndices(const ioda::ObsDataVector & //! Return the indices of observations that have failed quality control in //! at least one variable. //! -std::vector getFailedObservationIndices(const ioda::ObsDataVector & qcflags, +std::vector getFailedObservationIndices(const ioda::ObsDataVector & qc_flags, const eckit::mpi::Comm &comm, const std::vector &globalIdxFromLocalIdx) { - return getObservationIndicesWhere(qcflags, comm, globalIdxFromLocalIdx, + return getObservationIndicesWhere(qc_flags, comm, globalIdxFromLocalIdx, [](int qcFlag) { return qcFlag != 0; }); } @@ -294,11 +294,11 @@ std::vector getFailedObservationIndices(const ioda::ObsDataVector & //! Return the indices of observations whose quality control flag is set to \p flag in //! at least one variable. //! -std::vector getFlaggedObservationIndices(const ioda::ObsDataVector & qcflags, +std::vector getFlaggedObservationIndices(const ioda::ObsDataVector & qc_flags, const eckit::mpi::Comm &comm, const std::vector &globalIdxFromLocalIdx, int flag) { - return getObservationIndicesWhere(qcflags, comm, globalIdxFromLocalIdx, + return getObservationIndicesWhere(qc_flags, comm, globalIdxFromLocalIdx, [flag](int qcFlag) { return qcFlag == flag; }); } @@ -392,16 +392,17 @@ void expectVariablesRelativelyEqual(const ioda::ObsSpace &obsspace, void testFilters(size_t obsSpaceIndex, ioda::ObsSpace &obspace, const ObsTypeParameters ¶ms) { /// init QC and error - ioda::ObsDataVector obserr(obspace, obspace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags(obspace, obspace.obsvariables()); + std::shared_ptr> obserrfilter = + std::make_shared>(obspace, obspace.obsvariables(), "ObsError"); + std::shared_ptr> qc_flags_ = + std::make_shared>(obspace, obspace.obsvariables()); // Create filters and run preProcess ufo::ObsFilters filters(obspace, params.filtersParams.toConfiguration(), - qcflags, obserr); + qc_flags_, obserrfilter); filters.preProcess(); - /// call priorFilter and postFilter if hofx is available oops::Variables geovars = filters.requiredVars(); oops::ObsVariables diagvars = filters.requiredHdiagnostics(); @@ -449,7 +450,7 @@ void testFilters(size_t obsSpaceIndex, ioda::ObsSpace &obspace, diagvars += ybias.requiredHdiagnostics(); ufo::ObsDiagnostics diags(obspace, hop.locations(), diagvars); filters.priorFilter(gval); - hop.simulateObs(gval, hofx, ybias, qcflags, bias, diags); + hop.simulateObs(gval, hofx, ybias, *qc_flags_, bias, diags); filters.postFilter(gval, hofx, bias, diags); hofx.save("hofx"); } else if (geovars.size() > 0) { @@ -459,20 +460,20 @@ void testFilters(size_t obsSpaceIndex, ioda::ObsSpace &obspace, oops::Log::info() << "HofX or ObsOperator sections not provided for filters, " << "postFilter not called" << std::endl; /// apply the FinalCheck filter (which should always be run after all other filters). - runFinalCheck(obspace, qcflags, obserr); - obserr.mask(qcflags); + runFinalCheck(obspace, qc_flags_, obserrfilter); + obserrfilter->mask(*qc_flags_); } else { /// no need to run priorFilter or postFilter - oops::Log::info() << "GeoVaLs not required, HofX or ObsOperator sections not " - << "provided for filters, only preProcess was called" << std::endl; + oops::Log::info() << "GeoVaLs not required, HofX or ObsOperator sections not " << + "provided for filters, only preProcess was called" << std::endl; /// apply the FinalCheck filter (which should always be run after all other filters). - runFinalCheck(obspace, qcflags, obserr); - obserr.mask(qcflags); + runFinalCheck(obspace, qc_flags_, obserrfilter); + obserrfilter->mask(*qc_flags_); } - qcflags.save("EffectiveQC"); + qc_flags_->save("EffectiveQC"); const std::string errname = "EffectiveError"; - obserr.save(errname); + obserrfilter->save(errname); // Compare with known results bool atLeastOneBenchmarkFound = false; @@ -482,14 +483,14 @@ void testFilters(size_t obsSpaceIndex, ioda::ObsSpace &obspace, const std::vector &passedObsBenchmark = *params.passedObservationsBenchmark.value(); const std::vector passedObs = getPassedObservationIndices( - qcflags, obspace.comm(), obspace.index()); + *qc_flags_, obspace.comm(), obspace.index()); EXPECT_EQUAL(passedObs, passedObsBenchmark); } if (params.passedBenchmark.value() != boost::none) { atLeastOneBenchmarkFound = true; const size_t passedBenchmark = *params.passedBenchmark.value(); - size_t passed = numEqualTo(qcflags, ufo::QCflags::pass, + size_t passed = numEqualTo(*qc_flags_, ufo::QCflags::pass, *obspace.distribution()); EXPECT_EQUAL(passed, passedBenchmark); } @@ -499,14 +500,14 @@ void testFilters(size_t obsSpaceIndex, ioda::ObsSpace &obspace, const std::vector &failedObsBenchmark = *params.failedObservationsBenchmark.value(); const std::vector failedObs = getFailedObservationIndices( - qcflags, obspace.comm(), obspace.index()); + *qc_flags_, obspace.comm(), obspace.index()); EXPECT_EQUAL(failedObs, failedObsBenchmark); } if (params.failedBenchmark.value() != boost::none) { atLeastOneBenchmarkFound = true; const size_t failedBenchmark = *params.failedBenchmark.value(); - size_t failed = numNonzero(qcflags, *obspace.distribution()); + size_t failed = numNonzero(*qc_flags_, *obspace.distribution()); EXPECT_EQUAL(failed, failedBenchmark); } @@ -518,7 +519,7 @@ void testFilters(size_t obsSpaceIndex, ioda::ObsSpace &obspace, const std::vector &flaggedObsBenchmark = *params.flaggedObservationsBenchmark.value(); const std::vector flaggedObs = - getFlaggedObservationIndices(qcflags, obspace.comm(), + getFlaggedObservationIndices(*qc_flags_, obspace.comm(), obspace.index(), flag); EXPECT_EQUAL(flaggedObs, flaggedObsBenchmark); } @@ -526,7 +527,7 @@ void testFilters(size_t obsSpaceIndex, ioda::ObsSpace &obspace, if (params.flaggedBenchmark.value() != boost::none) { atLeastOneBenchmarkFound = true; const size_t flaggedBenchmark = *params.flaggedBenchmark.value(); - size_t flagged = numEqualTo(qcflags, flag, *obspace.distribution()); + size_t flagged = numEqualTo(*qc_flags_, flag, *obspace.distribution()); EXPECT_EQUAL(flagged, flaggedBenchmark); } } diff --git a/test/ufo/ObsLocalization.h b/test/ufo/ObsLocalization.h deleted file mode 100644 index fc508f537..000000000 --- a/test/ufo/ObsLocalization.h +++ /dev/null @@ -1,112 +0,0 @@ -/* - * (C) Copyright 2026 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#ifndef TEST_UFO_OBSLOCALIZATION_H_ -#define TEST_UFO_OBSLOCALIZATION_H_ - -#include -#include -#include -#include - -#define ECKIT_TESTING_SELF_REGISTER_CASES 0 - -#include "eckit/config/LocalConfiguration.h" -#include "eckit/geometry/Point3.h" -#include "eckit/testing/Test.h" - -#include "ioda/ObsSpace.h" - -#include "oops/mpi/mpi.h" -#include "oops/runs/Test.h" -#include "oops/util/Logger.h" - -#include "test/TestEnvironment.h" - -#include "ufo/instantiateObsLocFactory.h" -#include "ufo/obslocalization/ObsLocalization.h" - -namespace ufo { -namespace test { - -/// \brief Tests the point3/point3 computeLocalization method of ufo obs localizations. -/// -/// For each obs localization configured under "obs localizations" in the YAML, the test: -/// 1. Constructs the localization object. -/// 2. Loops over "point pairs" specified in the config. -/// 3. For each pair, calls computeLocalization(p1, p2) and compares to a -/// reference value. -/// NOTE this does not test the geometry iterator / obsvector localization.... -/// that wasn't Travis' job that day. -void testObsLocalizationPoint3() { - typedef ioda::ObsIterator Iterator_; - - instantiateObsLocFactory(); - - const eckit::Configuration & conf = ::test::TestEnvironment::config(); - const util::TimeWindow timeWindow(conf.getSubConfiguration("time window")); - const eckit::LocalConfiguration obsconf(conf, "obs space"); - ioda::ObsSpace obsspace(obsconf, oops::mpi::world(), timeWindow, oops::mpi::myself()); - - const std::vector locConfs = - conf.getSubConfigurations("obs localizations"); - - for (const auto & locConf : locConfs) { - const eckit::LocalConfiguration locSubConf(locConf, "localization"); - const std::string name = locSubConf.getString("localization method"); - oops::Log::info() << "Testing obs localization point3/point3: " << name << std::endl; - - ObsLocalization obsloc(locSubConf, obsspace); - - const std::vector pairConfs = - locConf.getSubConfigurations("point pair tests"); - - for (const auto & pairConf : pairConfs) { - const std::vector coords1 = pairConf.getDoubleVector("p1"); - const std::vector coords2 = pairConf.getDoubleVector("p2"); - ASSERT(coords1.size() == 3); - ASSERT(coords2.size() == 3); - const eckit::geometry::Point3 p1(coords1[0], coords1[1], coords1[2]); - const eckit::geometry::Point3 p2(coords2[0], coords2[1], coords2[2]); - - const double tol = pairConf.getDouble("tolerance"); - const double ref = pairConf.getDouble("reference value"); - const double result = obsloc.computeLocalization(p1, p2); - - oops::Log::info() << " p1=" << p1 << " p2=" << p2 - << " result=" << result << " ref=" << ref << std::endl; - EXPECT(std::abs(result - ref) <= tol); - } - } -} - -// ----------------------------------------------------------------------------- - -class ObsLocalization : public oops::Test { - public: - using oops::Test::Test; - virtual ~ObsLocalization() = default; - - private: - std::string testid() const override { return "ufo::test::ObsLocalization"; } - - void register_tests() const override { - std::vector& ts = eckit::testing::specification(); - - ts.emplace_back(CASE("ufo/ObsLocalization/testObsLocalizationPoint3") - { testObsLocalizationPoint3(); }); - } - - void clear() const override {} -}; - -// ----------------------------------------------------------------------------- - -} // namespace test -} // namespace ufo - -#endif // TEST_UFO_OBSLOCALIZATION_H_ diff --git a/test/ufo/ParallelPoissonDiskThinning.h b/test/ufo/ParallelPoissonDiskThinning.h index 093361e23..c6e3063b6 100644 --- a/test/ufo/ParallelPoissonDiskThinning.h +++ b/test/ufo/ParallelPoissonDiskThinning.h @@ -48,8 +48,10 @@ void testPoissonDiskThinning(const eckit::LocalConfiguration &conf, const eckit::LocalConfiguration obsSpaceConf(conf, "obs space"); ioda::ObsSpace obsspace(obsSpaceConf, oops::mpi::world(), timeWindow, oops::mpi::myself()); - ioda::ObsDataVector obserr(obsspace, obsspace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags(obsspace, obsspace.obsvariables()); + std::shared_ptr> obserr(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables(), "ObsError")); + std::shared_ptr> qcflags(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables())); eckit::LocalConfiguration filterConf(conf, "Poisson Disk Thinning"); ufo::PoissonDiskThinningParameters filterParameters; @@ -62,9 +64,9 @@ void testPoissonDiskThinning(const eckit::LocalConfiguration &conf, filter.preProcess(); - std::vector isObsRetained(qcflags.nlocs(), 0); - for (size_t i = 0; i < qcflags.nlocs(); ++i) - isObsRetained[i] = (qcflags[0][i] == ufo::QCflags::pass); + std::vector isObsRetained(qcflags->nlocs(), 0); + for (size_t i = 0; i < qcflags->nlocs(); ++i) + isObsRetained[i] = ((*qcflags)[0][i] == ufo::QCflags::pass); if (obsspace.distribution()->name() == "InefficientDistribution") { // PART 1: Verify that all ranks have retained the same observations. diff --git a/test/ufo/PoissonDiskThinning.h b/test/ufo/PoissonDiskThinning.h index 6b944dd3e..fbef79ec8 100644 --- a/test/ufo/PoissonDiskThinning.h +++ b/test/ufo/PoissonDiskThinning.h @@ -61,8 +61,10 @@ void testPoissonDiskThinning(const eckit::LocalConfiguration &conf, obsspace.put_db("MetaData", "thinningPriority", priorities); } - ioda::ObsDataVector obserr(obsspace, obsspace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags(obsspace, obsspace.obsvariables()); + std::shared_ptr> obserr(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables(), "ObsError")); + std::shared_ptr> qcflags(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables())); eckit::LocalConfiguration filterConf(conf, "Poisson Disk Thinning"); ufo::PoissonDiskThinningParameters filterParameters; @@ -78,8 +80,8 @@ void testPoissonDiskThinning(const eckit::LocalConfiguration &conf, const std::vector expectedThinnedObsIndices = conf.getUnsignedVector("expected_thinned_obs_indices"); std::vector thinnedObsIndices; - for (size_t i = 0; i < qcflags.nlocs(); ++i) - if (qcflags[0][i] == ufo::QCflags::thinned) + for (size_t i = 0; i < qcflags->nlocs(); ++i) + if ((*qcflags)[0][i] == ufo::QCflags::thinned) thinnedObsIndices.push_back(i); EXPECT_EQUAL(thinnedObsIndices, expectedThinnedObsIndices); } diff --git a/test/ufo/Reconditioning.h b/test/ufo/Reconditioning.h index 13d65c4fd..78897d8d5 100644 --- a/test/ufo/Reconditioning.h +++ b/test/ufo/Reconditioning.h @@ -27,7 +27,6 @@ #include "test/interface/ObsTestsFixture.h" #include "test/TestEnvironment.h" -#include "ufo/errors/ObsErrorBase.h" #include "ufo/errors/ObsErrorCrossVarCov.h" #include "ufo/errors/ObsErrorDiagonal.h" #include "ufo/errors/ObsErrorWithinGroupCov.h" @@ -53,24 +52,24 @@ void testNoReconditioning() { for (std::size_t jj = 0; jj < conf.size(); ++jj) { const eckit::LocalConfiguration rconf(conf[jj], "obs error"); - ufo::ObsErrorParametersWrapper Params; + ufo::ObsErrorCrossVarCovParameters Params; Params.validateAndDeserialize(rconf); - if (Params.errorParams().reconditioning.value().ReconMethod.value() + if (Params.reconditioning.value().ReconMethod.value() != ufo::ObsErrorReconditionerMethod::NORECONDITIONING) continue; const eckit::LocalConfiguration obsSpaceConf(conf[jj], "obs space"); ioda::ObsSpace obsspace(obsSpaceConf, oops::mpi::myself(), timeWindow, oops::mpi::myself()); - ObsErrorBase* R = ufo::ObsErrorFactory::create(Params.errorParams(), obsspace); - ObsErrorBase* RRecon = ufo::ObsErrorFactory::create(Params.errorParams(), obsspace); + ObsErrorCrossVarCov R(Params, obsspace, oops::mpi::myself()); + ObsErrorCrossVarCov RRecon(Params, obsspace, oops::mpi::myself()); oops::Log::info() << "Corr before:\n" << R << std::endl; ioda::ObsVector mask(obsspace, "ObsError"); - RRecon->update(mask); + RRecon.update(mask); oops::Log::info() << "Corr after:\n" << RRecon << std::endl; - const double rmseR = R->getRMSE(); - const double rmseRR = RRecon->getRMSE(); + const double rmseR = R.getRMSE(); + const double rmseRR = RRecon.getRMSE(); EXPECT(oops::is_close(rmseR, rmseRR, 1e-10)); } } @@ -92,24 +91,21 @@ void compareKnownOutput() { const eckit::LocalConfiguration rconf(conf[jj], "obs error"); const eckit::LocalConfiguration testconf(conf[jj], "obs error test"); - ufo::ObsErrorParametersWrapper Params; + ObsErrorCrossVarCovParameters Params; ReconditioningTestParameters TestParams; Params.validateAndDeserialize(rconf); TestParams.validateAndDeserialize(testconf); - ObsErrorBase* R = ufo::ObsErrorFactory::create(Params.errorParams(), obsspace); + ObsErrorCrossVarCov R(Params, obsspace, oops::mpi::myself()); ioda::ObsVector mask(obsspace, "ObsError"); ioda::ObsVector sample(obsspace, "ObsValue"); std::vector refVec = TestParams.refVec.value().value(); - // update method reconditions the ObsError matrix - R->update(mask); - // multiply method applies reconditioned ObsError matrix to the sample vector - R->multiply(sample); + R.update(mask); + R.multiply(sample); std::vector sampleVec; sample.maskAndSerialize(sample, sampleVec); oops::Log::info() << "R times sample vector: " << sampleVec << std::endl; oops::Log::info() << "Reference vector: " << refVec << std::endl << std::endl; - ASSERT(sampleVec.size() == refVec.size()); for (size_t i = 0; i < sampleVec.size(); ++i) { EXPECT(oops::is_close(sampleVec[i], refVec[i], 1e-5)); @@ -133,11 +129,11 @@ void testNoValidOptionSelected() { ioda::ObsSpace obsspace(obsSpaceConf, oops::mpi::myself(), timeWindow, oops::mpi::myself()); const eckit::LocalConfiguration rconf(conf[jj], "obs error"); - ufo::ObsErrorParametersWrapper Params; + ObsErrorCrossVarCovParameters Params; Params.validateAndDeserialize(rconf); const std::string msg = conf[jj].getString("expectExceptionWithMessage"); - EXPECT_THROWS_MSG(ufo::ObsErrorFactory::create(Params.errorParams(), obsspace), msg.c_str()); + EXPECT_THROWS_MSG(ObsErrorCrossVarCov R(Params, obsspace, oops::mpi::myself()), msg.c_str()); } } diff --git a/test/ufo/SampleAndReduceOverFieldOfView.h b/test/ufo/SampleAndReduceOverFieldOfView.h index 1f1308370..969ebed4d 100644 --- a/test/ufo/SampleAndReduceOverFieldOfView.h +++ b/test/ufo/SampleAndReduceOverFieldOfView.h @@ -226,6 +226,7 @@ void testReduceOverFieldOfView() { // Make up fake samples for 3 obs with {4,5,6} samples each const int nlocs = 3; + const int nsamples = 15; const std::vector> ranges({{{0, 4}, {4, 9}, {9, 15}}}); std::vector samples({{0.2, 0.3, 0.6, 0.9, 1.7, 1.2, 1.9, 4.2, 2.9, diff --git a/test/ufo/SampledLocations.h b/test/ufo/SampledLocations.h index 454b35f6e..3db26fb9e 100644 --- a/test/ufo/SampledLocations.h +++ b/test/ufo/SampledLocations.h @@ -172,6 +172,8 @@ void testFortranTimeMask() { // ----------------------------------------------------------------------------- /// Tests operator+= (concatenation of two SampledLocations) void testConcatenate() { + typedef SampledLocationsTestFixture Test_; + const eckit::LocalConfiguration conf(::test::TestEnvironment::config()); SampledLocations locs1(conf, oops::mpi::world()); SampledLocations locs2(conf, oops::mpi::world()); diff --git a/test/ufo/StuckCheck.h b/test/ufo/StuckCheck.h index 49bb53802..01d2e27f0 100644 --- a/test/ufo/StuckCheck.h +++ b/test/ufo/StuckCheck.h @@ -45,8 +45,10 @@ void testStuckCheck(const eckit::LocalConfiguration &conf) { obsspace.put_db("ObsValue", "pressure", airPressures); } - ioda::ObsDataVector obserr(obsspace, obsspace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags(obsspace, obsspace.obsvariables()); + std::shared_ptr> obserr(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables(), "ObsError")); + std::shared_ptr> qcflags(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables())); eckit::LocalConfiguration filterConf(conf, "Stuck Check"); ufo::StuckCheckParameters filterParameters; @@ -57,8 +59,8 @@ void testStuckCheck(const eckit::LocalConfiguration &conf) { const std::vector expectedRejectedObsIndices = conf.getUnsignedVector("expected_rejected_obs_indices"); std::vector rejectedObsIndices; - for (size_t i = 0; i < qcflags.nlocs(); ++i) - if (qcflags[0][i] == ufo::QCflags::track) + for (size_t i = 0; i < qcflags->nlocs(); ++i) + if ((*qcflags)[0][i] == ufo::QCflags::track) rejectedObsIndices.push_back(i); EXPECT_EQUAL(rejectedObsIndices, expectedRejectedObsIndices); } diff --git a/test/ufo/TemporalThinning.h b/test/ufo/TemporalThinning.h index fc4061df3..f63eafcbc 100644 --- a/test/ufo/TemporalThinning.h +++ b/test/ufo/TemporalThinning.h @@ -55,8 +55,10 @@ void testTemporalThinning(const eckit::LocalConfiguration &conf) { obsspace.put_db("MetaData", "priority", priorities); } - ioda::ObsDataVector obserr(obsspace, obsspace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags(obsspace, obsspace.obsvariables()); + std::shared_ptr> obserr(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables(), "ObsError")); + std::shared_ptr> qcflags(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables())); eckit::LocalConfiguration filterConf(conf, "TemporalThinning"); ufo::TemporalThinningParameters filterParameters; @@ -68,8 +70,8 @@ void testTemporalThinning(const eckit::LocalConfiguration &conf) { const std::vector expectedThinnedObsIndices = conf.getUnsignedVector("expected_thinned_obs_indices"); std::vector thinnedObsIndices; - for (size_t i = 0; i < qcflags.nlocs(); ++i) - if (qcflags[0][i] == ufo::QCflags::thinned) + for (size_t i = 0; i < qcflags->nlocs(); ++i) + if ((*qcflags)[0][i] == ufo::QCflags::thinned) thinnedObsIndices.push_back(i); EXPECT_EQUAL(thinnedObsIndices, expectedThinnedObsIndices); } diff --git a/test/ufo/TrackCheck.h b/test/ufo/TrackCheck.h index 838c5dcae..9c6b2454f 100644 --- a/test/ufo/TrackCheck.h +++ b/test/ufo/TrackCheck.h @@ -47,8 +47,10 @@ void testTrackCheck(const eckit::LocalConfiguration &conf) { obsspace.put_db("MetaData", "stationIdentification", stationIds); } - ioda::ObsDataVector obserr(obsspace, obsspace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags(obsspace, obsspace.obsvariables()); + std::shared_ptr> obserr(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables(), "ObsError")); + std::shared_ptr> qcflags(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables())); eckit::LocalConfiguration filterConf(conf, "Track Check"); ufo::TrackCheckParameters filterParameters; @@ -59,8 +61,8 @@ void testTrackCheck(const eckit::LocalConfiguration &conf) { const std::vector expectedRejectedObsIndices = conf.getUnsignedVector("expected_rejected_obs_indices"); std::vector rejectedObsIndices; - for (size_t i = 0; i < qcflags.nlocs(); ++i) - if (qcflags[0][i] == ufo::QCflags::track) + for (size_t i = 0; i < qcflags->nlocs(); ++i) + if ((*qcflags)[0][i] == ufo::QCflags::track) rejectedObsIndices.push_back(i); EXPECT_EQUAL(rejectedObsIndices, expectedRejectedObsIndices); } diff --git a/test/ufo/TrackCheckShip.h b/test/ufo/TrackCheckShip.h index 9a8587def..2a68d02b2 100644 --- a/test/ufo/TrackCheckShip.h +++ b/test/ufo/TrackCheckShip.h @@ -51,8 +51,10 @@ const boost::optional setupRunFilter( obsspace.put_db("ObsValue", "airTemperature", air_temperature); } - ioda::ObsDataVector obserr(obsspace, obsspace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags(obsspace, obsspace.obsvariables()); + std::shared_ptr> obserr(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables(), "ObsError")); + std::shared_ptr> qcflags(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables())); eckit::LocalConfiguration filterConf(conf, "Ship Track Check"); ufo::TrackCheckShipParameters filterParameters; @@ -60,8 +62,8 @@ const boost::optional setupRunFilter( ufo::TrackCheckShip filter(obsspace, filterParameters, qcflags, obserr); filter.preProcess(); if (filterConf.getBool("comparison test", false) && rejectedObsIndices) { - for (size_t i = 0; i < qcflags.nlocs(); ++i) - if (qcflags[0][i] == ufo::QCflags::track) + for (size_t i = 0; i < qcflags->nlocs(); ++i) + if ((*qcflags)[0][i] == ufo::QCflags::track) rejectedObsIndices->push_back(i); return boost::none; } else if (filterConf.getBool("unit testing mode", false)) { diff --git a/test/ufo/TrackCheckShipMainLoop.h b/test/ufo/TrackCheckShipMainLoop.h index 37c9fdfde..d1a54047c 100644 --- a/test/ufo/TrackCheckShipMainLoop.h +++ b/test/ufo/TrackCheckShipMainLoop.h @@ -41,8 +41,10 @@ void testFirstRejectionSimultaneousIncluded(const eckit::LocalConfiguration &con obsspace.put_db("MetaData", "stationIdentification", stationIds); } - ioda::ObsDataVector obserr(obsspace, obsspace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags(obsspace, obsspace.obsvariables()); + std::shared_ptr> obserr(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables(), "ObsError")); + std::shared_ptr> qcflags(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables())); eckit::LocalConfiguration filterConf(conf, "Ship Track Check"); ufo::TrackCheckShipParameters filterParameters; @@ -122,7 +124,7 @@ void testFirstRejectionSimultaneousIncluded(const eckit::LocalConfiguration &con } for (auto const& i : expectedFirstRejectionIndices) { - EXPECT_EQUAL(qcflags[0][i], ufo::QCflags::track); + EXPECT_EQUAL((*qcflags)[0][i], ufo::QCflags::track); // tests that the rejected observations are changing the qc flags correctly } }