From 5b0dbc9a977bf0100e53a0e0e1513e5b0793ffe6 Mon Sep 17 00:00:00 2001 From: Rudolf Weeber Date: Sun, 22 Mar 2026 12:47:09 +0100 Subject: [PATCH 01/11] Remove fallback code for ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM disabled find src/core -name '*.hpp' -o -name '*.cpp' -o -name '*.impl.hpp' | \ xargs grep -l ESPRESSO_SHARED_MEMORY_PARALLELISM | \ xargs -I{} unifdef -DESPRESSO_SHARED_MEMORY_PARALLELISM -t {} -o /dev/null --- src/core/aosoa_pack.hpp | 4 - src/core/cell_system/CellStructure.cpp | 9 -- src/core/cell_system/CellStructure.hpp | 22 +---- src/core/cell_system/particle_enumeration.hpp | 4 - .../CollisionDetection.cpp | 4 - src/core/communication.cpp | 12 --- src/core/communication.hpp | 2 - src/core/custom_verlet_list.hpp | 4 - src/core/electrostatics/elc.cpp | 6 -- src/core/electrostatics/elc.hpp | 4 - src/core/electrostatics/icc.cpp | 10 --- src/core/electrostatics/p3m_heffte.hpp | 16 +--- src/core/electrostatics/p3m_heffte.impl.hpp | 89 +------------------ src/core/forces.cpp | 51 ----------- src/core/forces_cabana.hpp | 4 - src/core/forces_inline.hpp | 2 - src/core/integrate.cpp | 2 - src/core/magnetostatics/dp3m_heffte.hpp | 16 +--- src/core/magnetostatics/dp3m_heffte.impl.hpp | 67 -------------- src/core/p3m/for_each_3d.hpp | 6 -- src/core/p3m/interpolation.hpp | 50 +---------- src/core/particle_reduction.hpp | 13 --- src/core/short_range_cabana.hpp | 4 - src/core/system/System.cpp | 12 --- src/core/system/System.hpp | 2 - .../unit_tests/field_layout_helpers_test.cpp | 4 - .../unit_tests/particle_reduction_test.cpp | 6 -- 27 files changed, 7 insertions(+), 418 deletions(-) diff --git a/src/core/aosoa_pack.hpp b/src/core/aosoa_pack.hpp index b095dd409c..b61a534d6d 100644 --- a/src/core/aosoa_pack.hpp +++ b/src/core/aosoa_pack.hpp @@ -21,8 +21,6 @@ #include -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM - #include "cell_system/CellStructure.hpp" #include @@ -119,5 +117,3 @@ struct CellStructure::AoSoA_pack { bool has_exclusion(std::size_t i) const { return flags(i) == uint8_t{1}; } }; - -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM diff --git a/src/core/cell_system/CellStructure.cpp b/src/core/cell_system/CellStructure.cpp index 908f3f58bd..36a23b729c 100644 --- a/src/core/cell_system/CellStructure.cpp +++ b/src/core/cell_system/CellStructure.cpp @@ -51,12 +51,10 @@ #include -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM #include #include #include #include -#endif #include #include @@ -77,14 +75,11 @@ #include CellStructure::~CellStructure() { -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM clear_local_properties(); // Kokkos handle can only be freed after all Cabana containers have been freed m_kokkos_handle.reset(); -#endif } -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM void CellStructure::clear_local_properties() { m_local_force.reset(); #ifdef ESPRESSO_ROTATION @@ -242,8 +237,6 @@ void CellStructure::set_index_map() { m_num_local_particles_cached = unique_particles.size(); } -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM - CellStructure::CellStructure(BoxGeometry const &box) : m_decomposition{std::make_unique(box)} {} @@ -520,7 +513,6 @@ void CellStructure::update_ghosts_and_resort_particle(unsigned data_parts) { } } -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM void CellStructure::parallel_for_each_particle_impl( std::span cells, ParticleUnaryOp &f) const { if (cells.size() > 1) { @@ -536,7 +528,6 @@ void CellStructure::parallel_for_each_particle_impl( [&](auto part_idx) { f(*(particles.begin() + part_idx)); }); } } -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM bool CellStructure::check_resort_required( Utils::Vector3d const &additional_offset) const { diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index 0c50ff4fbd..8bed75d701 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -62,7 +62,6 @@ #endif // forward declarations -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM namespace Kokkos { template class View; class HostSpace; @@ -80,7 +79,6 @@ class AoSoA; struct KokkosHandle; template class CustomVerletList; -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM template concept ParticleCallback = requires(Callable c, Particle &p) { @@ -168,7 +166,6 @@ struct EuclidianDistance { * be stored in separate structures. */ class CellStructure : public System::Leaf { -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM public: static constexpr auto vector_length = 1; struct AoSoA_pack; @@ -179,7 +176,6 @@ class CellStructure : public System::Leaf { using ListType = CustomVerletList; -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM private: /** The local id-to-particle index */ @@ -199,7 +195,6 @@ class CellStructure : public System::Leaf { /** @brief Verlet list skin. */ double m_verlet_skin = 0.; double m_verlet_reuse = 0.; -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM int m_cached_max_local_particle_id = 0; std::size_t m_num_local_particles_cached = 0; int m_max_id = 0; @@ -217,7 +212,6 @@ class CellStructure : public System::Leaf { /** The local id-to-index for aosoa data */ std::vector m_unique_particles; std::shared_ptr m_kokkos_handle; -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM public: CellStructure(BoxGeometry const &box); @@ -354,13 +348,7 @@ class CellStructure : public System::Leaf { } /** @brief whether to use parallel version of @ref for_each_local_particle */ - bool use_parallel_for_each_local_particle() const { -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM - return true; -#else - return false; -#endif - } + bool use_parallel_for_each_local_particle() const { return true; } /** * @brief Run a kernel on all local particles. @@ -368,12 +356,10 @@ class CellStructure : public System::Leaf { */ void for_each_local_particle(ParticleUnaryOp &&f, bool parallel = true) const { -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM if (parallel and use_parallel_for_each_local_particle()) { parallel_for_each_particle_impl(decomposition().local_cells(), f); return; } -#endif for (auto &p : local_particles()) { f(p); } @@ -403,10 +389,8 @@ class CellStructure : public System::Leaf { return decomposition().particle_to_cell(p); } -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM void parallel_for_each_particle_impl(std::span cells, ParticleUnaryOp &f) const; -#endif public: /** @@ -456,14 +440,12 @@ class CellStructure : public System::Leaf { * this node, or -1 if there are no particles on this node. */ int get_max_local_particle_id() const; -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM int get_cached_max_local_particle_id() const { return m_cached_max_local_particle_id; } std::size_t get_num_local_particles_cached() const { return m_num_local_particles_cached; } -#endif /** * @brief Remove all particles from the cell system. @@ -728,7 +710,6 @@ class CellStructure : public System::Leaf { } } -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM public: auto get_max_id() const { return m_max_id; } @@ -787,7 +768,6 @@ class CellStructure : public System::Leaf { inline void cell_list_loop(auto &&kernel) { kernel(m_decomposition->local_cells(), m_decomposition->box()); } -#endif private: /** Non-bonded pair loop with verlet lists. diff --git a/src/core/cell_system/particle_enumeration.hpp b/src/core/cell_system/particle_enumeration.hpp index 82482699d5..73c6cf2bd6 100644 --- a/src/core/cell_system/particle_enumeration.hpp +++ b/src/core/cell_system/particle_enumeration.hpp @@ -24,9 +24,7 @@ #include "Cell.hpp" #include "CellStructure.hpp" -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM #include -#endif #include #include @@ -43,7 +41,6 @@ template inline void enumerate_local_particles(CellStructure const &cs, Kernel &&kernel) { -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM if (cs.use_parallel_for_each_local_particle()) { auto const local_cells = cs.decomposition().local_cells(); @@ -66,7 +63,6 @@ inline void enumerate_local_particles(CellStructure const &cs, }); return; } -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM // Sequential fallback std::size_t index = 0; for (auto &p : cs.local_particles()) { diff --git a/src/core/collision_detection/CollisionDetection.cpp b/src/core/collision_detection/CollisionDetection.cpp index 3acb29006f..a2690bdac8 100644 --- a/src/core/collision_detection/CollisionDetection.cpp +++ b/src/core/collision_detection/CollisionDetection.cpp @@ -51,9 +51,7 @@ void CollisionDetection::initialize() { *m_protocol); } system.on_short_range_ia_change(); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM system.cell_structure->clear_local_properties(); -#endif } void CollisionDetection::set_protocol( @@ -66,9 +64,7 @@ void CollisionDetection::unset_protocol() { m_protocol = nullptr; auto &system = get_system(); system.on_short_range_ia_change(); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM system.cell_structure->clear_local_properties(); -#endif } } // namespace CollisionDetection diff --git a/src/core/communication.cpp b/src/core/communication.cpp index e5d30ee4ff..7be6a9c4bb 100644 --- a/src/core/communication.cpp +++ b/src/core/communication.cpp @@ -31,11 +31,9 @@ #include #endif -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM #include #include #include -#endif #include #include @@ -57,19 +55,15 @@ #include #include -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM struct KokkosHandle { KokkosHandle() { Kokkos::initialize(); } ~KokkosHandle() { Kokkos::finalize(); } }; -#endif boost::mpi::communicator comm_cart; Communicator communicator{}; std::unique_ptr communication_environment{}; -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM std::shared_ptr kokkos_handle{}; -#endif int this_node = -1; [[maybe_unused]] static auto get_env_variable(char const *const name) { @@ -93,12 +87,10 @@ CommunicationEnvironment::CommunicationEnvironment() CommunicationEnvironment::CommunicationEnvironment( std::shared_ptr mpi_env) : m_mpi_env{std::move(mpi_env)} { -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM auto const num_threads_env = get_env_variable("OMP_NUM_THREADS"); if (not num_threads_env or num_threads_env->empty()) { omp_set_num_threads(1); } -#endif m_is_mpi_gpu_aware = false; @@ -140,16 +132,12 @@ CommunicationEnvironment::CommunicationEnvironment( fft_on_program_start(); #endif -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM kokkos_handle = std::make_shared(); -#endif } CommunicationEnvironment::~CommunicationEnvironment() { -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM Kokkos::fence(); kokkos_handle.reset(); -#endif #ifdef ESPRESSO_WALBERLA walberla::mpi_deinit(); diff --git a/src/core/communication.hpp b/src/core/communication.hpp index 322c6a710f..4dd02f2c0c 100644 --- a/src/core/communication.hpp +++ b/src/core/communication.hpp @@ -59,10 +59,8 @@ extern int this_node; /** The communicator */ extern boost::mpi::communicator comm_cart; -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM struct KokkosHandle; extern std::shared_ptr kokkos_handle; -#endif class CommunicationEnvironment { std::shared_ptr m_mpi_env; diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index 0de3c6d83a..3a6dc2ec96 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -21,8 +21,6 @@ #include -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM - #include #include @@ -224,5 +222,3 @@ class Cabana::NeighborList -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM #include -#endif #include #include @@ -1128,12 +1126,8 @@ void charge_assign(elc_data const &elc, CoulombP3M &solver, solver.prepare_fft_mesh(protocol == ChargeProtocol::BOTH or protocol == ChargeProtocol::IMAGE); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM // multi-threading -> cache sizes must be equal to the number of particles auto constexpr include_neutral_particles = true; -#else - auto constexpr include_neutral_particles = false; -#endif for (auto zipped : p_q_pos_range) { auto const p_q = boost::get<0>(zipped); diff --git a/src/core/electrostatics/elc.hpp b/src/core/electrostatics/elc.hpp index 0ca479d053..8fd9f09440 100644 --- a/src/core/electrostatics/elc.hpp +++ b/src/core/electrostatics/elc.hpp @@ -116,7 +116,6 @@ struct elc_data { /** The space that is finally left. */ double space_box; -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM /// pairwise contributions from lower and upper layers void dielectric_layers_contribution(BoxGeometry const &box_geo, std::size_t p1, std::size_t p2, @@ -140,7 +139,6 @@ struct elc_data { kernel(q_eff, d); } } -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM /// pairwise contributions from lower and upper layers void dielectric_layers_contribution(BoxGeometry const &box_geo, @@ -281,7 +279,6 @@ struct ElectrostaticLayerCorrection base_solver); } -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM /** @brief Calculate short-range pair energy correction. */ double pair_energy_correction(std::size_t p1, std::size_t p2, auto &aosoa, double q1q2) const { @@ -307,7 +304,6 @@ struct ElectrostaticLayerCorrection } return energy; } -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM /** @brief Calculate short-range pair energy correction. */ double pair_energy_correction(Utils::Vector3d const &pos1, diff --git a/src/core/electrostatics/icc.cpp b/src/core/electrostatics/icc.cpp index cb792fa668..4bf3ee056c 100644 --- a/src/core/electrostatics/icc.cpp +++ b/src/core/electrostatics/icc.cpp @@ -48,10 +48,8 @@ #include #include -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM #include #include -#endif #include #include @@ -77,9 +75,7 @@ static void force_calc_icc( auto const reset_kernel = [](Particle &p) { p.force_and_torque() = {}; }; cell_structure.for_each_local_particle(reset_kernel); cell_structure.for_each_ghost_particle(reset_kernel); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM cell_structure.reset_local_force(); -#endif // calc ICC forces cell_structure.non_bonded_loop( @@ -120,11 +116,9 @@ void ICCStar::iteration() { auto const elc_kernel = coulomb.pair_force_elc_kernel(); icc_cfg.citeration = 0; -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM using execution_space = Kokkos::DefaultExecutionSpace; auto const &unique_particles = cell_structure.get_unique_particles(); auto const &local_force = cell_structure.get_local_force(); -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM auto global_max_rel_diff = 0.; @@ -135,7 +129,6 @@ void ICCStar::iteration() { force_calc_icc(cell_structure, kernel, elc_kernel); system.coulomb.calc_long_range_force(); cell_structure.ghosts_reduce_forces(); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM // force reduction int num_threads = execution_space().concurrency(); kokkos_parallel_range_for>( @@ -149,7 +142,6 @@ void ICCStar::iteration() { } }); Kokkos::fence(); -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM auto max_rel_diff = 0.; @@ -217,10 +209,8 @@ void ICCStar::iteration() { /* Update charges on ghosts. */ cell_structure.ghosts_update(Cells::DATA_PART_PROPERTIES); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM // refresh local properties update_aosoa_charges(cell_structure); -#endif icc_cfg.citeration++; diff --git a/src/core/electrostatics/p3m_heffte.hpp b/src/core/electrostatics/p3m_heffte.hpp index 777862be8b..bf7af121df 100644 --- a/src/core/electrostatics/p3m_heffte.hpp +++ b/src/core/electrostatics/p3m_heffte.hpp @@ -36,10 +36,8 @@ #include #include -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM #include #include -#endif #include #include @@ -86,7 +84,6 @@ struct CoulombP3MState : public P3MStateCommon { std::array, 3> rs_E_fields_no_halo; p3m_send_mesh halo_comm; std::shared_ptr> fft; -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM Kokkos::View rs_charge_density_kokkos; @@ -95,7 +92,6 @@ struct CoulombP3MState : public P3MStateCommon { rs_charge_density_kokkos = decltype(rs_charge_density_kokkos)( "CoulombP3MState::rs_charge_density_kokkos", 0, 0); } -#endif }; #ifdef ESPRESSO_CUDA @@ -111,10 +107,8 @@ struct CoulombP3MHeffte : public CoulombP3M { CoulombP3MStateClass &p3m; private: -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM // kokkos handle must outlive kokkos data structures from other class members std::shared_ptr m_kokkos_handle; -#endif std::unique_ptr p3m_state_ptr; TuningParameters tuning; bool m_is_tuned; @@ -131,10 +125,8 @@ struct CoulombP3MHeffte : public CoulombP3M { CoulombP3MHeffte(std::unique_ptr &&p3m_state, TuningParameters tuning_params, double prefactor) : CoulombP3M(p3m_state->params), p3m{*p3m_state}, -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM - m_kokkos_handle{::kokkos_handle}, -#endif - p3m_state_ptr{std::move(p3m_state)}, tuning{std::move(tuning_params)} { + m_kokkos_handle{::kokkos_handle}, p3m_state_ptr{std::move(p3m_state)}, + tuning{std::move(tuning_params)} { if (tuning.timings <= 0) { throw std::domain_error("Parameter 'timings' must be > 0"); @@ -142,9 +134,7 @@ struct CoulombP3MHeffte : public CoulombP3M { m_is_tuned = not p3m.params.tuning; p3m.params.tuning = false; set_prefactor(prefactor); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM p3m.init_labels(); -#endif } void init() override { @@ -217,13 +207,11 @@ struct CoulombP3MHeffte : public CoulombP3M { p3m.inter_weights.reset(p3m.params.cao); } p3m.rs_charge_density.resize(p3m.local_mesh.size); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM using execution_space = Kokkos::DefaultExecutionSpace; auto const num_threads = execution_space().concurrency(); Kokkos::realloc(Kokkos::WithoutInitializing, p3m.rs_charge_density_kokkos, num_threads, p3m.local_mesh.size); Kokkos::deep_copy(p3m.rs_charge_density_kokkos, FloatType{0}); -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM std::ranges::fill(p3m.rs_charge_density, FloatType{0}); } diff --git a/src/core/electrostatics/p3m_heffte.impl.hpp b/src/core/electrostatics/p3m_heffte.impl.hpp index 3e3145e0c0..16de511231 100644 --- a/src/core/electrostatics/p3m_heffte.impl.hpp +++ b/src/core/electrostatics/p3m_heffte.impl.hpp @@ -77,10 +77,8 @@ #include #include -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM #include #include -#endif #include #include @@ -368,7 +366,6 @@ template struct AssignCharge { this->operator()(p3m, q, weights); } -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM void operator()(auto &p3m, auto &cell_structure) { using CoulombP3MState = std::remove_reference_t; using value_type = CoulombP3MState::value_type; @@ -405,17 +402,6 @@ template struct AssignCharge { }); Kokkos::fence(); } -#else // ESPRESSO_SHARED_MEMORY_PARALLELISM - void operator()(auto &p3m, auto const &p_q_pos_range) { - for (auto zipped : p_q_pos_range) { - auto const p_q = boost::get<0>(zipped); - if (p_q != 0.0) { - auto const &p_pos = boost::get<1>(zipped); - this->operator()(p3m, p_q, p_pos, p3m.inter_weights); - } - } - } -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM }; } // namespace @@ -423,18 +409,8 @@ template void CoulombP3MHeffte::charge_assign() { prepare_fft_mesh(true); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM Utils::integral_parameter( p3m.params.cao, p3m, *get_system().cell_structure); -#else // ESPRESSO_SHARED_MEMORY_PARALLELISM - auto const &system = get_system(); - auto const particles = system.cell_structure->local_particles(); - auto p_q_range = ParticlePropertyRange::charge_range(particles); - auto p_pos_range = ParticlePropertyRange::pos_range(particles); - - Utils::integral_parameter( - p3m.params.cao, p3m, boost::combine(p_q_range, p_pos_range)); -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM } template @@ -452,12 +428,7 @@ void CoulombP3MHeffte::assign_charge( namespace { template struct AssignForces { void operator()(auto &p3m, auto force_prefac, -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM - CellStructure &cell_structure -#else - auto const &p_q_force_range -#endif - ) const { + CellStructure &cell_structure) const { assert(cao == p3m.inter_weights.cao()); @@ -472,17 +443,12 @@ template struct AssignForces { force[2u] += w * double(p3m.rs_E_fields[2u][ind]); }); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM auto const thread_id = omp_get_thread_num(); p_force(p_index, thread_id, 0) -= pref * force[0]; p_force(p_index, thread_id, 1) -= pref * force[1]; p_force(p_index, thread_id, 2) -= pref * force[2]; -#else - p_force -= pref * force; -#endif }; -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM auto const n_part = cell_structure.count_local_particles(); auto const &aosoa = cell_structure.get_aosoa(); auto &local_force = cell_structure.get_local_force(); @@ -492,24 +458,10 @@ template struct AssignForces { kernel(pref, local_force, p_index); } }); -#else // ESPRESSO_SHARED_MEMORY_PARALLELISM - /* charged particle counter */ - std::size_t p_index{0ul}; - - for (auto zipped : p_q_force_range) { - auto p_q = boost::get<0>(zipped); - if (p_q != 0.) { - auto &p_force = boost::get<1>(zipped); - kernel(p_q * force_prefac, p_force, p_index); - ++p_index; - } - } -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM } }; } // namespace -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM inline auto calc_dipole_moment(boost::mpi::communicator const &comm, auto const &cs, auto const &box_geo) { auto const local_dip = reduce_over_local_particles( @@ -520,19 +472,6 @@ inline auto calc_dipole_moment(boost::mpi::communicator const &comm, [](Utils::Vector3d &a, Utils::Vector3d const &b) { a = a + b; }); return boost::mpi::all_reduce(comm, local_dip, std::plus<>()); } -#else // ESPRESSO_SHARED_MEMORY_PARALLELISM -inline auto calc_dipole_moment(boost::mpi::communicator const &comm, - auto const &p_q_unfolded_pos_range) { - auto const local_dip = - boost::accumulate(p_q_unfolded_pos_range, Utils::Vector3d{}, - [](Utils::Vector3d const &dip, auto const &q_pos) { - auto const p_q = boost::get<0>(q_pos); - auto const &p_unfolded_pos = boost::get<1>(q_pos); - return dip + p_q * p_unfolded_pos; - }); - return boost::mpi::all_reduce(comm, local_dip, std::plus<>()); -} -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM template void CoulombP3MHeffte::long_range_kernel( kernel_ks_charge_density(); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM auto const &local_force = cell_structure.get_local_force(); auto const &aosoa = cell_structure.get_aosoa(); -#else - auto const particles = cell_structure.local_particles(); - auto p_q_range = ParticlePropertyRange::charge_range(particles); - auto p_force_range = ParticlePropertyRange::force_range(particles); - auto p_unfolded_pos_range = - ParticlePropertyRange::unfolded_pos_range(particles, box_geo); -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM // The dipole moment is only needed if we don't have metallic boundaries auto const box_dipole = (p3m.params.epsilon != P3M_EPSILON_METALLIC) ? std::make_optional(calc_dipole_moment( -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM comm_cart, cell_structure, box_geo)) -#else - comm_cart, - boost::combine(p_q_range, - p_unfolded_pos_range))) -#endif : std::nullopt; auto const volume = box_geo.volume(); auto const pref = @@ -742,11 +667,7 @@ double CoulombP3MHeffte::long_range_kernel( // assign particle forces auto const force_prefac = prefactor / volume; -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM auto &particle_data = cell_structure; -#else - auto const particle_data = boost::combine(p_q_range, p_force_range); -#endif Utils::integral_parameter( p3m.params.cao, p3m, force_prefac, particle_data); @@ -754,7 +675,6 @@ double CoulombP3MHeffte::long_range_kernel( // Eq. (3.19) @cite deserno00b if (box_dipole) { auto const dm = prefactor * pref * box_dipole.value(); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM auto const n_part = cell_structure.count_local_particles(); kokkos_parallel_range_for( "AssignForcesBoxDipole", std::size_t{0u}, n_part, @@ -765,13 +685,6 @@ double CoulombP3MHeffte::long_range_kernel( local_force(p_index, thread_id, 1) -= q * dm[1]; local_force(p_index, thread_id, 2) -= q * dm[2]; }); -#else // ESPRESSO_SHARED_MEMORY_PARALLELISM - for (auto zipped : boost::combine(p_q_range, p_force_range)) { - auto p_q = boost::get<0>(zipped); - auto &p_force = boost::get<1>(zipped); - p_force -= p_q * dm; - } -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM } } diff --git a/src/core/forces.cpp b/src/core/forces.cpp index c7b0398864..f19d639196 100644 --- a/src/core/forces.cpp +++ b/src/core/forces.cpp @@ -57,9 +57,7 @@ #include #endif -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM #include -#endif #include #include @@ -125,9 +123,7 @@ static void init_forces_and_thermostat(System::System const &system) { #endif } }); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM cell_structure.reset_local_force(); -#endif // Initialize ghost forces (unchanged) cell_structure.ghosts_reset_forces(); @@ -153,7 +149,6 @@ static void reinit_dip_fld(CellStructure const &cell_structure) { } #endif -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM static ForcesKernel create_cabana_neighbor_kernel( System::System const &system, Utils::Vector3d *virial, auto const &elc_kernel, auto const &coulomb_kernel, @@ -243,7 +238,6 @@ static void reduce_cabana_forces_and_torques(System::System const &system, } #endif } -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM void System::System::calculate_forces() { #ifdef ESPRESSO_CALIPER @@ -305,10 +299,8 @@ void System::System::calculate_forces() { dipoles.cutoff(), collision_detection_cutoff}; -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM update_cabana_state(*cell_structure, verlet_criterion, get_interaction_range(), propagation->integ_switch); -#endif #ifdef ESPRESSO_ELECTROSTATICS if (coulomb.impl->extension) { update_icc_particles(); @@ -328,7 +320,6 @@ void System::System::calculate_forces() { CALI_MARK_END("calc_long_range_forces"); #endif -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM #ifdef ESPRESSO_CALIPER CALI_MARK_BEGIN("cabana_short_range"); #endif @@ -359,48 +350,6 @@ void System::System::calculate_forces() { CALI_MARK_END("cabana_short_range"); #endif -#else // ESPRESSO_SHARED_MEMORY_PARALLELISM - -#ifdef ESPRESSO_CALIPER - CALI_MARK_BEGIN("serial_short_range"); -#endif - - auto pair_kernel = [coulomb_kernel_ptr = get_ptr(coulomb_kernel), - dipoles_kernel_ptr = get_ptr(dipoles_kernel), - elc_kernel_ptr = get_ptr(elc_kernel), - coulomb_u_kernel_ptr = get_ptr(coulomb_u_kernel), - &nonbonded_ias = *nonbonded_ias, - &thermostat = *thermostat, &bonded_ias = *bonded_ias, - virial, -#ifdef ESPRESSO_COLLISION_DETECTION - &collision_detection = *collision_detection, -#endif - &box_geo = *box_geo, - system_max_cutoff2 = Utils::sqr(maximal_cutoff())]( - Particle &p1, Particle &p2, Distance const &d) { - if (d.dist2 > system_max_cutoff2) - return; - auto const &ia_params = nonbonded_ias.get_ia_param(p1.type(), p2.type()); - add_non_bonded_pair_force( - p1, p2, d.vec21, sqrt(d.dist2), d.dist2, p1.q() * p2.q(), ia_params, - thermostat, box_geo, bonded_ias, virial, coulomb_kernel_ptr, - dipoles_kernel_ptr, elc_kernel_ptr, coulomb_u_kernel_ptr); -#ifdef ESPRESSO_COLLISION_DETECTION - if (not collision_detection.is_off()) { - collision_detection.detect_collision(p1, p2, d.dist2); - } -#endif - }; - - short_range_loop(bond_kernel, pair_kernel, *cell_structure, maximal_cutoff(), - bonded_ias->maximal_cutoff(), verlet_criterion); - -#ifdef ESPRESSO_CALIPER - CALI_MARK_END("serial_short_range"); -#endif - -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM - constraints->add_forces(particles, get_sim_time()); oif_global->calculate_forces(); diff --git a/src/core/forces_cabana.hpp b/src/core/forces_cabana.hpp index c7319f7448..ad183433bf 100644 --- a/src/core/forces_cabana.hpp +++ b/src/core/forces_cabana.hpp @@ -21,8 +21,6 @@ #include -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM - #include "aosoa_pack.hpp" #include "forces_inline.hpp" @@ -324,5 +322,3 @@ struct ForcesKernel { #endif } }; - -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM diff --git a/src/core/forces_inline.hpp b/src/core/forces_inline.hpp index 9131004ce3..3c1b6f5e2e 100644 --- a/src/core/forces_inline.hpp +++ b/src/core/forces_inline.hpp @@ -73,9 +73,7 @@ #include #include -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM ESPRESSO_ATTR_ALWAYS_INLINE -#endif inline Utils::Vector3d calc_central_radial_force(IA_parameters const &ia_params, Utils::Vector3d const &d, double const dist) { diff --git a/src/core/integrate.cpp b/src/core/integrate.cpp index c9a41b9528..4d22e247a4 100644 --- a/src/core/integrate.cpp +++ b/src/core/integrate.cpp @@ -865,12 +865,10 @@ int System::System::integrate(int n_steps, int reuse_forces) { if (caught_error) { return INTEG_ERROR_RUNTIME; } -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM if (boost::mpi::all_reduce(::comm_cart, not cell_structure->use_verlet_list, std::logical_or<>())) { cell_structure->use_verlet_list = false; } -#endif return integrated_steps; } diff --git a/src/core/magnetostatics/dp3m_heffte.hpp b/src/core/magnetostatics/dp3m_heffte.hpp index 6e53b64a80..2e9f7ed669 100644 --- a/src/core/magnetostatics/dp3m_heffte.hpp +++ b/src/core/magnetostatics/dp3m_heffte.hpp @@ -36,10 +36,8 @@ #include -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM #include #include -#endif #include #include @@ -137,7 +135,6 @@ struct DipolarP3MState : public P3MStateCommon { void resize_heffte_buffers(); #endif // ESPRESSO_DP3M_HEFFTE_CROSS_CHECKS -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM Kokkos::View rs_fields_kokkos; @@ -151,7 +148,6 @@ struct DipolarP3MState : public P3MStateCommon { rs_fields_kokkos = decltype(rs_fields_kokkos)( "DipolarP3MState::rs_fields_kokkos", 0, 0, 0); } -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM }; template @@ -167,10 +163,8 @@ struct DipolarP3MHeffte : public DipolarP3M { DipolarP3MStateClass &dp3m; private: -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM // kokkos handle must outlive kokkos data structures from other class members std::shared_ptr m_kokkos_handle; -#endif /** @brief Dipolar P3M meshes and FFT algorithm. */ std::unique_ptr dp3m_impl; TuningParameters tuning; @@ -180,10 +174,8 @@ struct DipolarP3MHeffte : public DipolarP3M { DipolarP3MHeffte(std::unique_ptr &&dp3m_state, TuningParameters tuning_params, double prefactor) : DipolarP3M(dp3m_state->params), dp3m{*dp3m_state}, -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM - m_kokkos_handle{::kokkos_handle}, -#endif - dp3m_impl{std::move(dp3m_state)}, tuning{std::move(tuning_params)} { + m_kokkos_handle{::kokkos_handle}, dp3m_impl{std::move(dp3m_state)}, + tuning{std::move(tuning_params)} { if (tuning.timings <= 0) { throw std::domain_error("Parameter 'timings' must be > 0"); @@ -194,9 +186,7 @@ struct DipolarP3MHeffte : public DipolarP3M { m_is_tuned = not dp3m.params.tuning; dp3m.params.tuning = false; set_prefactor(prefactor); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM dp3m.init_labels(); -#endif } void init() override { @@ -233,13 +223,11 @@ struct DipolarP3MHeffte : public DipolarP3M { private: void prepare_fft_mesh() { dp3m.inter_weights.reset(dp3m.params.cao); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM using execution_space = Kokkos::DefaultExecutionSpace; auto const num_threads = execution_space().concurrency(); Kokkos::realloc(Kokkos::WithoutInitializing, dp3m.rs_fields_kokkos, num_threads, 3, dp3m.local_mesh.size); Kokkos::deep_copy(dp3m.rs_fields_kokkos, FloatType{0}); -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM for (auto &rs_mesh_field : dp3m.mesh.rs_fields) { std::ranges::fill(rs_mesh_field, FloatType{0}); } diff --git a/src/core/magnetostatics/dp3m_heffte.impl.hpp b/src/core/magnetostatics/dp3m_heffte.impl.hpp index 6b4f118853..463e07b317 100644 --- a/src/core/magnetostatics/dp3m_heffte.impl.hpp +++ b/src/core/magnetostatics/dp3m_heffte.impl.hpp @@ -61,10 +61,8 @@ #include #include -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM #include #include -#endif #include #include @@ -181,7 +179,6 @@ void DipolarP3MHeffte::init_cpu_kernels() { namespace { template struct AssignDipole { -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM void operator()(auto &dp3m, auto &cell_structure) { using DipolarP3MState = std::remove_reference_t; using value_type = DipolarP3MState::value_type; @@ -226,29 +223,6 @@ template struct AssignDipole { }); Kokkos::fence(); } -#else // ESPRESSO_SHARED_MEMORY_PARALLELISM - void operator()(auto &dp3m, Utils::Vector3d const &real_pos, - Utils::Vector3d const &dip) const { - using DipolarP3MState = std::remove_reference_t; - using value_type = DipolarP3MState::value_type; - auto constexpr memory_order = Utils::MemoryOrder::ROW_MAJOR; - auto const weights = p3m_calculate_interpolation_weights( - real_pos, dp3m.params.ai, dp3m.local_mesh); - p3m_interpolate( - dp3m.local_mesh, weights, [&dip, &dp3m](int ind, double w) { - dp3m.mesh.rs_fields[0u][ind] += value_type(w * dip[0u]); - dp3m.mesh.rs_fields[1u][ind] += value_type(w * dip[1u]); - dp3m.mesh.rs_fields[2u][ind] += value_type(w * dip[2u]); -#ifdef ESPRESSO_DP3M_HEFFTE_CROSS_CHECKS - dp3m.heffte.rs_dipole_density[0u][ind] += value_type(w * dip[0u]); - dp3m.heffte.rs_dipole_density[1u][ind] += value_type(w * dip[1u]); - dp3m.heffte.rs_dipole_density[2u][ind] += value_type(w * dip[2u]); -#endif - }); - - dp3m.inter_weights.template store(weights); - } -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM }; } // namespace @@ -256,17 +230,8 @@ template void DipolarP3MHeffte::dipole_assign() { prepare_fft_mesh(); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM Utils::integral_parameter( dp3m.params.cao, dp3m, *get_system().cell_structure); -#else // ESPRESSO_SHARED_MEMORY_PARALLELISM - for (auto const &p : get_system().cell_structure->local_particles()) { - if (p.dipm() != 0.) { - Utils::integral_parameter( - dp3m.params.cao, dp3m, p.pos(), p.calc_dip()); - } - } -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM } namespace { @@ -287,17 +252,12 @@ template struct AssignTorques { }); auto const torque = vector_product(pref, E); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM auto const thread_id = omp_get_thread_num(); p_torque(p_index, thread_id, 0) -= torque[0]; p_torque(p_index, thread_id, 1) -= torque[1]; p_torque(p_index, thread_id, 2) -= torque[2]; -#else - p_torque -= torque; -#endif }; -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM auto const n_part = dp3m.inter_weights.size(); auto const &unique_particles = cell_structure.get_unique_particles(); auto &local_torque = cell_structure.get_local_torque(); @@ -308,17 +268,6 @@ template struct AssignTorques { kernel(p.calc_dip() * prefac, local_torque, p_index); } }); -#else // ESPRESSO_SHARED_MEMORY_PARALLELISM - /* magnetic particle index */ - auto p_index = std::size_t{0ul}; - - for (auto &p : cell_structure.local_particles()) { - if (p.dipm() != 0.) { - kernel(p.calc_dip() * prefac, p.torque(), p_index); - ++p_index; - } - } -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM } }; @@ -340,15 +289,10 @@ template struct AssignForcesDip { E[2u] += w * double(dp3m.mesh.rs_fields[2u][ind]); }); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM auto const thread_id = omp_get_thread_num(); p_force(p_index, thread_id, d_rs) += pref * E; -#else - p_force[d_rs] += pref * E; -#endif }; -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM auto const n_part = dp3m.inter_weights.size(); auto const &unique_particles = cell_structure.get_unique_particles(); auto &local_force = cell_structure.get_local_force(); @@ -359,17 +303,6 @@ template struct AssignForcesDip { kernel(p.calc_dip() * prefac, local_force, p_index); } }); -#else // ESPRESSO_SHARED_MEMORY_PARALLELISM - /* magnetic particle index */ - auto p_index = std::size_t{0ul}; - - for (auto &p : cell_structure.local_particles()) { - if (p.dipm() != 0.) { - kernel(p.calc_dip() * prefac, p.force(), p_index); - ++p_index; - } - } -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM } }; } // namespace diff --git a/src/core/p3m/for_each_3d.hpp b/src/core/p3m/for_each_3d.hpp index cc96f48d5e..b5f8c7a601 100644 --- a/src/core/p3m/for_each_3d.hpp +++ b/src/core/p3m/for_each_3d.hpp @@ -23,9 +23,7 @@ #include -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM #include -#endif #include #include @@ -123,14 +121,12 @@ void for_each_3d_order(detail::IndexVectorConcept auto &&start, } } -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM /** @brief Mapping between ESPResSo and Kokkos tags for memory order */ template using LayoutIterate = std::conditional_t< Order == Utils::MemoryOrder::COLUMN_MAJOR, std::integral_constant, std::integral_constant>; -#endif /** * @brief Run a kernel(index_3d, linear_index) over the given 3d range with @@ -139,7 +135,6 @@ using LayoutIterate = std::conditional_t< template void for_each_3d_lin(detail::IndexVectorConcept auto &&start, detail::IndexVectorConcept auto &&stop, Kernel &&kernel) { -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM if (Kokkos::num_threads() > 1) { auto const size = stop - start; constexpr Kokkos::Iterate iter = LayoutIterate::value; @@ -153,7 +148,6 @@ void for_each_3d_lin(detail::IndexVectorConcept auto &&start, }); return; } -#endif int linear_loop_index = 0u; if constexpr (memory_order == Utils::MemoryOrder::ROW_MAJOR) { diff --git a/src/core/p3m/interpolation.hpp b/src/core/p3m/interpolation.hpp index 04112495ee..afccc66e43 100644 --- a/src/core/p3m/interpolation.hpp +++ b/src/core/p3m/interpolation.hpp @@ -26,9 +26,7 @@ #include #include -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM #include -#endif #include #include @@ -69,38 +67,22 @@ template struct InterpolationWeightsView { class p3m_interpolation_cache { int m_cao = 0; /** Charge fractions for mesh assignment. */ -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM Kokkos::View ca_frac; std::vector ca_frac_spillover; -#else - std::vector ca_frac; -#endif /** index of first mesh point for charge assignment. */ -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM Kokkos::View ca_fmp; std::vector ca_fmp_spillover; -#else - std::vector ca_fmp; -#endif public: -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM p3m_interpolation_cache() : ca_frac("p3m_interpolation_cache::ca_frac", 0, 0), ca_fmp("p3m_interpolation_cache::ca_fmp", 0, 0) {} -#endif /** * @brief Number of points in the cache. * @return Number of points currently in the cache. */ - auto size() const { -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM - return ca_fmp.size() + ca_fmp_spillover.size(); -#else - return ca_fmp.size(); -#endif - } + auto size() const { return ca_fmp.size() + ca_fmp_spillover.size(); } /** * @brief Charge assignment order the weights are for. @@ -115,7 +97,6 @@ class p3m_interpolation_cache { */ void zfill(std::size_t size) { auto const size_frac = size * static_cast(m_cao * 3); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM if (ca_fmp.size() != size or ca_frac.size() != size_frac) { Kokkos::realloc(Kokkos::WithoutInitializing, ca_fmp, size); Kokkos::realloc(Kokkos::WithoutInitializing, ca_frac, size_frac); @@ -125,12 +106,6 @@ class p3m_interpolation_cache { // data must be contiguous in memory assert(size == 0 or (std::distance(&ca_fmp(0), &ca_fmp(size - 1)) == size - 1)); -#else - assert(ca_frac.empty()); - assert(ca_fmp.empty()); - ca_fmp.resize(size); - ca_frac.resize(size_frac); -#endif } /** @@ -143,19 +118,11 @@ class p3m_interpolation_cache { template void store(InterpolationWeights const &weights) { assert(cao == m_cao); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM ca_fmp_spillover.emplace_back(weights.ind); auto it = std::back_inserter(ca_frac_spillover); std::ranges::copy(weights.w_x, it); std::ranges::copy(weights.w_y, it); std::ranges::copy(weights.w_z, it); -#else - ca_fmp.emplace_back(weights.ind); - auto it = std::back_inserter(ca_frac); - std::ranges::copy(weights.w_x, it); - std::ranges::copy(weights.w_y, it); - std::ranges::copy(weights.w_z, it); -#endif } /** @@ -179,11 +146,7 @@ class p3m_interpolation_cache { std::copy_n(src.w_z.data(), cao, dst); }; -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM ca_fmp(p_index) = weights.ind; -#else - ca_fmp[p_index] = weights.ind; -#endif auto const offset = p_index * static_cast(cao * 3); copy_weights(weights, ca_frac.data() + offset); } @@ -206,7 +169,6 @@ class p3m_interpolation_cache { int index = -1; double const *ptr = nullptr; -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM if (p_index >= ca_fmp.size()) { p_index -= ca_fmp.size(); index = ca_fmp_spillover[p_index]; @@ -217,11 +179,6 @@ class p3m_interpolation_cache { auto const offset = p_index * static_cast(cao * 3); ptr = std::as_const(ca_frac).data() + offset; } -#else - index = ca_fmp[p_index]; - auto const offset = p_index * static_cast(cao * 3); - ptr = std::as_const(ca_frac).data() + offset; -#endif std::span w_x(ptr, cao); std::advance(ptr, cao); std::span w_y(ptr, cao); @@ -237,15 +194,10 @@ class p3m_interpolation_cache { */ void reset(int cao) { m_cao = cao; -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM Kokkos::realloc(Kokkos::WithoutInitializing, ca_fmp, 0ul); Kokkos::realloc(Kokkos::WithoutInitializing, ca_frac, 0ul); ca_frac_spillover.clear(); ca_fmp_spillover.clear(); -#else - ca_frac.clear(); - ca_fmp.clear(); -#endif } }; diff --git a/src/core/particle_reduction.hpp b/src/core/particle_reduction.hpp index 2850feeb93..5858f00249 100644 --- a/src/core/particle_reduction.hpp +++ b/src/core/particle_reduction.hpp @@ -24,9 +24,7 @@ #include "Particle.hpp" #include "cell_system/CellStructure.hpp" -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM #include -#endif #include #include @@ -43,8 +41,6 @@ using AddPartialResultKernel = template using ReductionOp = std::function; -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM - /** @brief Custom reduction in the form required by Kokkos */ template class KokkosReducer { public: @@ -84,8 +80,6 @@ make_kokkos_reducer(Kernel k, ReductionOp reduce_op) { return KokkosReducer(k, reduce_op); } -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM - } // namespace Reduction /** @brief performs a reduction over all particles @@ -105,7 +99,6 @@ ResultType reduce_over_local_particles( ResultType result{}; -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM auto const &cells = cs.decomposition().local_cells(); if (cells.size() > 1) { // parallel loop over cells auto reducer = Reduction::make_kokkos_reducer( @@ -129,10 +122,4 @@ ResultType reduce_over_local_particles( Kokkos::parallel_reduce( // loop over particles "reduce_on_local_particle", particles.size(), reducer, result); return result; -#else // ESPRESSO_SHARED_MEMORY_PARALLELISM - for (auto const &p : cs.local_particles()) { - add_partial(result, p); - } - return result; -#endif } diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 23f128745a..432de3fb53 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -21,8 +21,6 @@ #include -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM - #include "cell_system/CellStructure.hpp" #include "aosoa_pack.hpp" @@ -316,5 +314,3 @@ void cabana_short_range(auto const &bond_kernel, auto const &forces_kernel, #endif } } - -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM diff --git a/src/core/system/System.cpp b/src/core/system/System.cpp index 87b9738cda..644b780980 100644 --- a/src/core/system/System.cpp +++ b/src/core/system/System.cpp @@ -73,9 +73,7 @@ System::System(Private) { box_geo = std::make_shared(); local_geo = std::make_shared(); cell_structure = std::make_shared(*box_geo); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM cell_structure->set_kokkos_handle(::kokkos_handle); -#endif #ifdef ESPRESSO_CUDA gpu = std::make_shared(); #endif @@ -358,9 +356,7 @@ void System::on_particle_change() { /* the particle information is no longer valid */ invalidate_fetch_cache(); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM cell_structure->clear_local_properties(); -#endif } void System::on_particle_charge_change() { @@ -392,9 +388,7 @@ void System::update_dependent_particles() { #ifdef ESPRESSO_ELECTROSTATICS if (has_icc_enabled()) { -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM rebuild_aosoa(); -#endif update_icc_particles(); } #endif @@ -420,12 +414,9 @@ void System::on_observable_calc() { #endif clear_particle_node(); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM rebuild_aosoa(); -#endif } -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM void System::rebuild_aosoa() { #ifdef ESPRESSO_COLLISION_DETECTION auto const collision_detection_cutoff = collision_detection->cutoff(); @@ -443,7 +434,6 @@ void System::rebuild_aosoa() { update_cabana_state(*cell_structure, verlet_criterion, get_interaction_range(), propagation->integ_switch); } -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM void System::on_lees_edwards_change() { lb.on_lees_edwards_change(); } @@ -520,9 +510,7 @@ void System::on_integration_start() { } invalidate_fetch_cache(); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM cell_structure->clear_local_properties(); -#endif #ifdef ESPRESSO_ADDITIONAL_CHECKS if (!Utils::Mpi::all_compare(::comm_cart, cell_structure->use_verlet_list)) { diff --git a/src/core/system/System.hpp b/src/core/system/System.hpp index 3628f28e38..cfa15d44c4 100644 --- a/src/core/system/System.hpp +++ b/src/core/system/System.hpp @@ -137,9 +137,7 @@ class System : public std::enable_shared_from_this { /** @brief Rebuild cell lists. Use e.g. after a skin change. */ void rebuild_cell_structure(); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM void rebuild_aosoa(); -#endif /** @brief Calculate the maximal cutoff of all interactions. */ double maximal_cutoff() const; diff --git a/src/core/unit_tests/field_layout_helpers_test.cpp b/src/core/unit_tests/field_layout_helpers_test.cpp index cc8ffbf06a..df0fdbcde4 100644 --- a/src/core/unit_tests/field_layout_helpers_test.cpp +++ b/src/core/unit_tests/field_layout_helpers_test.cpp @@ -23,9 +23,7 @@ #include "p3m/field_layout_helpers.hpp" -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM #include -#endif #include #include @@ -36,10 +34,8 @@ #include struct GlobalConfig { -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM GlobalConfig() { Kokkos::initialize(); } ~GlobalConfig() { Kokkos::finalize(); } -#endif }; BOOST_TEST_GLOBAL_CONFIGURATION(GlobalConfig); diff --git a/src/core/unit_tests/particle_reduction_test.cpp b/src/core/unit_tests/particle_reduction_test.cpp index 455c4fc9cb..c738d8c753 100644 --- a/src/core/unit_tests/particle_reduction_test.cpp +++ b/src/core/unit_tests/particle_reduction_test.cpp @@ -55,11 +55,7 @@ struct GlobalConfig : public EspressoCoreGlobalConfig { // Decorator to skip tests if shared-memory parallelism isn't compiled in boost::test_tools::assertion_result has_shm(boost::unit_test::test_unit_id) { -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM return true; -#else - return false; -#endif } BOOST_TEST_GLOBAL_CONFIGURATION(GlobalConfig); @@ -69,7 +65,6 @@ auto const reduce_op = [](T &a, T const &b) { a = a + b; }; BOOST_TEST_DECORATOR(*boost::unit_test::precondition(has_shm)) BOOST_AUTO_TEST_CASE(test_make_kokkos_reduction) { -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM auto &system = System::get_system(); auto const &cell_structure = *system.cell_structure; auto const &cells = cell_structure.decomposition().local_cells(); @@ -90,7 +85,6 @@ BOOST_AUTO_TEST_CASE(test_make_kokkos_reduction) { // so make sure that both results are equal. BOOST_CHECK_EQUAL(res, ref); } -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM } BOOST_AUTO_TEST_CASE(test_reduce_over_local_particles) { From 03dee06580e68bd6b8e3c27e3df7f782b76666c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Grad?= Date: Thu, 2 Apr 2026 00:07:53 +0200 Subject: [PATCH 02/11] Make OpenMP a hard dependency --- .github/workflows/push_pull.yml | 7 +- .gitlab-ci.yml | 6 +- CMakeLists.txt | 130 ++++++++---------- cmake/espresso_cmake_config.cmakein | 2 - cmake/espresso_unit_test.cmake | 21 +-- doc/sphinx/installation.rst | 4 - doc/sphinx/running.rst | 5 +- maintainer/CI/build_cmake.sh | 2 - src/config/features.def | 1 - src/core/CMakeLists.txt | 17 +-- src/core/bond_breakage/bond_breakage.cpp | 6 +- src/core/bond_breakage/bond_breakage.hpp | 4 +- src/core/bond_forces_kokkos.hpp | 4 - src/core/cell_system/CellStructure.cpp | 4 - src/core/cell_system/LocalBondState.cpp | 3 - src/core/cell_system/LocalBondState.hpp | 5 +- src/core/collision_detection/utils.hpp | 2 - src/core/forces_inline.hpp | 6 - src/core/integrate.cpp | 4 - src/core/p3m/CMakeLists.txt | 17 +-- src/core/unit_tests/CMakeLists.txt | 40 +++--- .../EspressoSystemStandAlone_test.cpp | 2 - src/python/espressomd/CMakeLists.txt | 18 ++- src/walberla_bridge/CMakeLists.txt | 27 ++-- src/walberla_bridge/tests/CMakeLists.txt | 2 +- testsuite/python/CMakeLists.txt | 5 +- testsuite/python/caliper.py | 28 ++-- testsuite/python/cell_system.py | 2 +- 28 files changed, 136 insertions(+), 238 deletions(-) diff --git a/.github/workflows/push_pull.yml b/.github/workflows/push_pull.yml index 0954cb9d37..53cb5ec778 100644 --- a/.github/workflows/push_pull.yml +++ b/.github/workflows/push_pull.yml @@ -48,7 +48,6 @@ jobs: with_gsl: 'true' with_walberla: 'true' with_walberla_avx: 'false' - with_shared_memory_parallelism: 'true' debian: runs-on: ubuntu-latest @@ -85,7 +84,6 @@ jobs: with_gsl: 'false' with_scafacos: 'false' with_stokesian_dynamics: 'false' - with_shared_memory_parallelism: 'false' make_check_unit_tests: 'true' make_check_python: 'false' OMPI_ALLOW_RUN_AS_ROOT: 1 @@ -127,9 +125,9 @@ jobs: with_gsl: 'false' with_scafacos: 'false' with_stokesian_dynamics: 'false' - with_shared_memory_parallelism: 'false' make_check_unit_tests: 'false' make_check_python: 'false' + cmake_params: '-D ESPRESSO_TEST_NT=1' OMPI_ALLOW_RUN_AS_ROOT: 1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 @@ -172,5 +170,4 @@ jobs: with_walberla: 'true' with_walberla_avx: 'false' with_stokesian_dynamics: 'true' - with_shared_memory_parallelism: 'false' - cmake_params: '-D CMAKE_CXX_STANDARD=23 -D CMAKE_INCLUDE_PATH=/usr/include/mpich-x86_64 -D CMAKE_PREFIX_PATH=/usr/lib64/mpich/lib/' + cmake_params: '-D CMAKE_CXX_STANDARD=23 -D CMAKE_INCLUDE_PATH=/usr/include/mpich-x86_64 -D CMAKE_PREFIX_PATH=/usr/lib64/mpich/lib/ -D ESPRESSO_TEST_NT=1' diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3aa4576b66..baf0146044 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -47,7 +47,7 @@ style: - cd build - cp ../style.patch . - cp ../maintainer/configs/maxset.hpp myconfig.hpp - - cmake .. -D ESPRESSO_BUILD_WITH_CUDA=ON -D ESPRESSO_BUILD_WITH_FFTW=ON -D ESPRESSO_BUILD_WITH_GSL=ON -D ESPRESSO_BUILD_WITH_HDF5=ON -D ESPRESSO_BUILD_WITH_SCAFACOS=ON -D ESPRESSO_BUILD_WITH_WALBERLA=ON -D ESPRESSO_BUILD_WITH_STOKESIAN_DYNAMICS=ON -D ESPRESSO_BUILD_WITH_CALIPER=ON -D ESPRESSO_BUILD_WITH_FPE=ON -D ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM=ON -D ESPRESSO_BUILD_WITH_NLOPT=ON + - cmake .. -D ESPRESSO_BUILD_WITH_CUDA=ON -D ESPRESSO_BUILD_WITH_FFTW=ON -D ESPRESSO_BUILD_WITH_GSL=ON -D ESPRESSO_BUILD_WITH_HDF5=ON -D ESPRESSO_BUILD_WITH_SCAFACOS=ON -D ESPRESSO_BUILD_WITH_WALBERLA=ON -D ESPRESSO_BUILD_WITH_STOKESIAN_DYNAMICS=ON -D ESPRESSO_BUILD_WITH_CALIPER=ON -D ESPRESSO_BUILD_WITH_FPE=ON -D ESPRESSO_BUILD_WITH_NLOPT=ON - sh ../maintainer/CI/dox_warnings.sh tags: - espresso @@ -146,7 +146,6 @@ clang-sanitizer: with_scafacos: 'true' with_walberla: 'true' with_stokesian_dynamics: 'true' - with_shared_memory_parallelism: 'true' with_nlopt: 'true' with_coverage: 'false' with_coverage_python: 'false' @@ -193,7 +192,6 @@ cuda12-coverage: with_scafacos: 'true' with_walberla: 'true' with_stokesian_dynamics: 'true' - with_shared_memory_parallelism: 'true' with_nlopt: 'true' script: - bash maintainer/CI/build_cmake.sh @@ -221,7 +219,6 @@ cuda12-maxset: with_walberla: 'true' with_walberla_avx: 'true' with_stokesian_dynamics: 'true' - with_shared_memory_parallelism: 'true' with_nlopt: 'true' script: - bash maintainer/CI/build_cmake.sh @@ -367,7 +364,6 @@ installation: with_walberla: 'true' with_walberla_avx: 'true' with_stokesian_dynamics: 'true' - with_shared_memory_parallelism: 'true' with_nlopt: 'true' srcdir: '${CI_PROJECT_DIR}' build_type: 'Release' diff --git a/CMakeLists.txt b/CMakeLists.txt index 3c64721542..49a1316b4b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -108,8 +108,6 @@ option(ESPRESSO_BUILD_WITH_WALBERLA "Build with waLBerla support" ON) option(ESPRESSO_BUILD_WITH_WALBERLA_AVX "Build waLBerla kernels with AVX2 vectorization" ${ESPRESSO_BUILD_WITH_WALBERLA_AVX_DEFAULT}) -option(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM - "Build with shared memory parallelism support" OFF) option(ESPRESSO_BUILD_BENCHMARKS "Enable benchmarks" OFF) option(ESPRESSO_BUILD_WITH_VALGRIND "Build with Valgrind instrumentation" OFF) option(ESPRESSO_BUILD_WITH_CALIPER "Build with Caliper instrumentation" OFF) @@ -438,13 +436,6 @@ target_compile_options( $<$:-Wno-cast-function-type> $<$:-diag-disable=592> $<$:-Wno-global-constructors> - $<$,$>:-Wno-format-nonliteral> - $<$,$>:-Wno-float-conversion> - $<$,$>:-Wno-implicit-int-float-conversion> - $<$,$>:-Wno-implicit-float-conversion> - $<$,$>:-Wno-tautological-constant-compare> - $<$,$>:-Wno-ctad-maybe-unsupported> - $<$,$>:-Wno-extra-semi> $<$,$>:-Wno-extra-semi> $<$,$>:-Wno-extra-semi> $<$,$>:-Wno-cast-qual> @@ -613,10 +604,7 @@ endif() # if(ESPRESSO_BUILD_WITH_FFTW) - if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) - list(APPEND FFTW3_COMPONENTS omp) - endif() - find_package(fftw3 REQUIRED COMPONENTS ${FFTW3_COMPONENTS}) + find_package(fftw3 REQUIRED COMPONENTS omp) if(NOT EXISTS ${FETCHCONTENT_BASE_DIR}/heffte-src) find_package(Heffte 2.4.1 QUIET) @@ -649,62 +637,66 @@ if(ESPRESSO_BUILD_WITH_FFTW) endif() endif() -if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) - find_package(OpenMP REQUIRED COMPONENTS CXX) +find_package(OpenMP 4.5 REQUIRED COMPONENTS CXX) - if(NOT EXISTS ${FETCHCONTENT_BASE_DIR}/kokkos-src) - find_package(Kokkos 4.6 QUIET) - endif() - if(NOT DEFINED Kokkos_FOUND OR NOT ${Kokkos_FOUND}) - # cmake-format: off - FetchContent_Declare( - kokkos - GIT_REPOSITORY https://github.com/kokkos/kokkos.git - GIT_TAG 5.0.2 - OVERRIDE_FIND_PACKAGE - ) - # cmake-format: on - set(BUILD_SHARED_LIBS ON) - set(CMAKE_SHARED_LIBRARY_PREFIX "lib") - set(Kokkos_ENABLE_SERIAL ON CACHE BOOL "") - set(Kokkos_ENABLE_OPENMP ON CACHE BOOL "") - set(Kokkos_ENABLE_IMPL_VIEW_LEGACY ON CACHE BOOL "") - set(Kokkos_ENABLE_COMPLEX_ALIGN ON CACHE BOOL "") - set(Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION ON CACHE BOOL "") - set(Kokkos_ENABLE_HWLOC ON CACHE BOOL "") - set(Kokkos_ENABLE_DEPRECATED_CODE_5 OFF CACHE BOOL "") - set(Kokkos_ARCH_NATIVE ON CACHE BOOL "") - FetchContent_MakeAvailable(kokkos) - set(BUILD_SHARED_LIBS ${ESPRESSO_BUILD_SHARED_LIBS_DEFAULT}) - set(CMAKE_SHARED_LIBRARY_PREFIX "${ESPRESSO_SHARED_LIBRARY_PREFIX}") - install(TARGETS kokkos LIBRARY DESTINATION "${ESPRESSO_INSTALL_LIBDIR}") - # install all kokkos shared objects - get_target_property(ESPRESSO_KOKKOS_LIBS kokkos INTERFACE_LINK_LIBRARIES) - foreach(target_name IN LISTS ESPRESSO_KOKKOS_LIBS) - get_target_property(target_type ${target_name} TYPE) - if(${target_type} STREQUAL "SHARED_LIBRARY" AND ${target_name} MATCHES - "^kokkos[a-zA-Z0-9_]+$") - install(TARGETS ${target_name} - LIBRARY DESTINATION "${ESPRESSO_INSTALL_LIBDIR}") - endif() - endforeach() - endif() +if(NOT EXISTS ${FETCHCONTENT_BASE_DIR}/kokkos-src) + find_package(Kokkos 4.6 QUIET) +endif() +if(NOT DEFINED Kokkos_FOUND OR NOT ${Kokkos_FOUND}) + # cmake-format: off + FetchContent_Declare( + kokkos + GIT_REPOSITORY https://github.com/kokkos/kokkos.git + GIT_TAG 5.0.2 + OVERRIDE_FIND_PACKAGE + ) + # cmake-format: on + set(BUILD_SHARED_LIBS ON) + set(CMAKE_SHARED_LIBRARY_PREFIX "lib") + set(Kokkos_ENABLE_SERIAL ON CACHE BOOL "") + set(Kokkos_ENABLE_OPENMP ON CACHE BOOL "") + set(Kokkos_ENABLE_IMPL_VIEW_LEGACY ON CACHE BOOL "") + set(Kokkos_ENABLE_COMPLEX_ALIGN ON CACHE BOOL "") + set(Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION ON CACHE BOOL "") + set(Kokkos_ENABLE_HWLOC ON CACHE BOOL "") + set(Kokkos_ENABLE_DEPRECATED_CODE_5 OFF CACHE BOOL "") + set(Kokkos_ARCH_NATIVE ON CACHE BOOL "") + FetchContent_MakeAvailable(kokkos) + set(BUILD_SHARED_LIBS ${ESPRESSO_BUILD_SHARED_LIBS_DEFAULT}) + set(CMAKE_SHARED_LIBRARY_PREFIX "${ESPRESSO_SHARED_LIBRARY_PREFIX}") + install(TARGETS kokkos LIBRARY DESTINATION "${ESPRESSO_INSTALL_LIBDIR}") + # install all kokkos shared objects + get_target_property(ESPRESSO_KOKKOS_LIBS kokkos INTERFACE_LINK_LIBRARIES) + foreach(target_name IN LISTS ESPRESSO_KOKKOS_LIBS) + get_target_property(target_type ${target_name} TYPE) + if(${target_type} STREQUAL "SHARED_LIBRARY" AND ${target_name} MATCHES + "^kokkos[a-zA-Z0-9_]+$") + install(TARGETS ${target_name} + LIBRARY DESTINATION "${ESPRESSO_INSTALL_LIBDIR}") + endif() + endforeach() + # mark kokkos headers as system headers to disable compiler diagnostics + set_property( + TARGET kokkos APPEND + PROPERTY INTERFACE_SYSTEM_INCLUDE_DIRECTORIES + $) + set_target_properties(kokkos PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "") +endif() - if(NOT EXISTS ${FETCHCONTENT_BASE_DIR}/cabana-src) - find_package(Cabana 0.7.0 QUIET) - endif() - if(NOT DEFINED Cabana_FOUND OR NOT ${Cabana_FOUND}) - # cmake-format: off - FetchContent_Declare( - cabana - GIT_REPOSITORY https://github.com/ECP-copa/Cabana.git - GIT_TAG e76c1a1 # 0.7.0 with patches - PATCH_COMMAND patch -p0 < ${CMAKE_CURRENT_SOURCE_DIR}/cmake/cabana.patch - ) - # cmake-format: on - set(Cabana_REQUIRE_HEFFTE ${ESPRESSO_BUILD_WITH_FFTW} CACHE BOOL "") - FetchContent_MakeAvailable(cabana) - endif() +if(NOT EXISTS ${FETCHCONTENT_BASE_DIR}/cabana-src) + find_package(Cabana 0.7.0 QUIET) +endif() +if(NOT DEFINED Cabana_FOUND OR NOT ${Cabana_FOUND}) + # cmake-format: off + FetchContent_Declare( + cabana + GIT_REPOSITORY https://github.com/ECP-copa/Cabana.git + GIT_TAG e76c1a1 # 0.7.0 with patches + PATCH_COMMAND patch -p0 < ${CMAKE_CURRENT_SOURCE_DIR}/cmake/cabana.patch + ) + # cmake-format: on + set(Cabana_REQUIRE_HEFFTE ${ESPRESSO_BUILD_WITH_FFTW} CACHE BOOL "") + FetchContent_MakeAvailable(cabana) endif() # We need the parallel hdf5 version! @@ -980,9 +972,7 @@ if(ESPRESSO_BUILD_WITH_WALBERLA) endif() endif() set(WALBERLA_BUILD_WITH_FFTW off CACHE BOOL "") - if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) - set(WALBERLA_BUILD_WITH_OPENMP on CACHE BOOL "") - endif() + set(WALBERLA_BUILD_WITH_OPENMP on CACHE BOOL "") set(WALBERLA_BUILD_WITH_FASTMATH off CACHE BOOL "") set(BUILD_SHARED_LIBS OFF) FetchContent_MakeAvailable(walberla) diff --git a/cmake/espresso_cmake_config.cmakein b/cmake/espresso_cmake_config.cmakein index 1bd76a3f03..113f1e735d 100644 --- a/cmake/espresso_cmake_config.cmakein +++ b/cmake/espresso_cmake_config.cmakein @@ -23,8 +23,6 @@ #cmakedefine ESPRESSO_BUILD_WITH_FPE -#cmakedefine ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM - #define PACKAGE_NAME "${PROJECT_NAME}" /** diff --git a/cmake/espresso_unit_test.cmake b/cmake/espresso_unit_test.cmake index d0a44f30f6..dfbec9bf84 100644 --- a/cmake/espresso_unit_test.cmake +++ b/cmake/espresso_unit_test.cmake @@ -64,11 +64,10 @@ function(espresso_unit_test_register) if(NOT DEFINED TEST_TARGET) set(TEST_TARGET ${TEST_NAME}) endif() - if(NOT DEFINED TEST_NUM_PROC AND "${TEST_DEPENDS}" MATCHES "(^|;)([Bb]oost::mpi|MPI::MPI_CXX)($|;)") + if(NOT DEFINED TEST_NUM_PROC) set(TEST_NUM_PROC 1) endif() - # If NUM_PROC is given, set up MPI parallel test case - if(TEST_NUM_PROC) + if(TEST_NUM_PROC GREATER 1) if(${TEST_NUM_PROC} GREATER ${ESPRESSO_TEST_NP}) set(TEST_NUM_PROC ${ESPRESSO_TEST_NP}) endif() @@ -79,6 +78,9 @@ function(espresso_unit_test_register) ${MPIEXEC_POSTFLAGS}) else() add_test(NAME ${TEST_NAME} COMMAND ${TEST_TARGET}) + if(ESPRESSO_MPIEXEC_GUARD_SINGLETON_NUMA) + list(APPEND TEST_ENV_VARIABLES "OMPI_MCA_hwloc_base_binding_policy=none") + endif() endif() if(NOT DEFINED TEST_NUM_THREADS) set(TEST_NUM_THREADS 2) @@ -86,6 +88,7 @@ function(espresso_unit_test_register) if(${TEST_NUM_THREADS} GREATER ${ESPRESSO_TEST_NT}) set(TEST_NUM_THREADS ${ESPRESSO_TEST_NT}) endif() + list(APPEND TEST_ENV_VARIABLES "OMP_PROC_BIND=false" "OMP_NUM_THREADS=${TEST_NUM_THREADS}") if(ESPRESSO_WARNINGS_ARE_ERRORS) set(SANITIZERS_HALT_ON_ERROR "halt_on_error=1") @@ -95,17 +98,7 @@ function(espresso_unit_test_register) list(APPEND TEST_ENV_VARIABLES "UBSAN_OPTIONS=suppressions=${CMAKE_SOURCE_DIR}/maintainer/CI/ubsan.supp:${SANITIZERS_HALT_ON_ERROR}:print_stacktrace=1") list(APPEND TEST_ENV_VARIABLES "ASAN_OPTIONS=${SANITIZERS_HALT_ON_ERROR}:detect_leaks=0:allocator_may_return_null=1") list(APPEND TEST_ENV_VARIABLES "MSAN_OPTIONS=${SANITIZERS_HALT_ON_ERROR}") - if(NOT DEFINED TEST_NUM_PROC AND ESPRESSO_MPIEXEC_GUARD_SINGLETON_NUMA) - list(APPEND TEST_ENV_VARIABLES "OMPI_MCA_hwloc_base_binding_policy=none") - endif() - list(APPEND TEST_ENV_VARIABLES "OMP_PROC_BIND=false" "OMP_NUM_THREADS=${TEST_NUM_THREADS}") - set(TEST_NUM_CORES 1) - if(DEFINED TEST_NUM_PROC) - set(TEST_NUM_CORES ${TEST_NUM_PROC}) - endif() - if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) - math(EXPR TEST_NUM_CORES "${TEST_NUM_CORES} * ${TEST_NUM_THREADS}") - endif() + math(EXPR TEST_NUM_CORES "${TEST_NUM_PROC} * ${TEST_NUM_THREADS}") set_tests_properties( ${TEST_NAME} PROPERTIES ENVIRONMENT "${TEST_ENV_VARIABLES}" PROCESSORS ${TEST_NUM_CORES} diff --git a/doc/sphinx/installation.rst b/doc/sphinx/installation.rst index be670925c1..1af69a1811 100644 --- a/doc/sphinx/installation.rst +++ b/doc/sphinx/installation.rst @@ -791,9 +791,6 @@ using a CMake flag (see :ref:`Options and Variables`). - ``STOKESIAN_DYNAMICS``: enable the Stokesian Dynamics propagator (see :ref:`Stokesian Dynamics`). Requires BLAS and LAPACK. -- ``SHARED_MEMORY_PARALLELISM``: enable shared-memory parallelism - (OpenMP, Kokkos, Cabana) - - ``CALIPER``, ``VALGRIND``, ``FPE``: enable various instrumentation tools (see :ref:`Instrumentation`) @@ -957,7 +954,6 @@ The following options control features from external libraries: * ``ESPRESSO_BUILD_WITH_SCAFACOS``: Build with ScaFaCoS support. * ``ESPRESSO_BUILD_WITH_GSL``: Build with GSL support. * ``ESPRESSO_BUILD_WITH_STOKESIAN_DYNAMICS`` Build with Stokesian Dynamics support. -* ``ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM``: Build with shared-memory parallelism support (OpenMP, Cabana, Kokkos, etc.) * ``ESPRESSO_BUILD_WITH_WALBERLA``: Build with waLBerla support. * ``ESPRESSO_BUILD_WITH_WALBERLA_AVX``: Build waLBerla with AVX kernels instead of regular kernels. * ``ESPRESSO_BUILD_WITH_PYTHON``: Build with the Python interface. diff --git a/doc/sphinx/running.rst b/doc/sphinx/running.rst index dc9e134237..028a439a54 100644 --- a/doc/sphinx/running.rst +++ b/doc/sphinx/running.rst @@ -262,9 +262,8 @@ extra arguments are passed to the ``mpiexec`` program. On cluster computers, it might be necessary to load the MPI library with ``module load openmpi`` or similar. -On modern NUMA architectures, |es| can leverage shared-memory parallelism -(SMP) using the `OpenMP `__ programming model. -This is enabled via the CMake option ``-D SHARED_MEMORY_PARALLELISM=ON``. +|es| leverages shared-memory parallelism (SMP) using +the `OpenMP `__ programming model. To run a simulation with 4 OpenMP threads, use the following syntax: .. code-block:: bash diff --git a/maintainer/CI/build_cmake.sh b/maintainer/CI/build_cmake.sh index d04ddc2f6e..3f2f5dd2a4 100755 --- a/maintainer/CI/build_cmake.sh +++ b/maintainer/CI/build_cmake.sh @@ -107,7 +107,6 @@ set_default_value with_asan false set_default_value with_static_analysis false set_default_value with_caliper false set_default_value with_fpe false -set_default_value with_shared_memory_parallelism false set_default_value myconfig "default" set_default_value build_procs ${default_build_procs} set_default_value check_procs ${default_check_procs} @@ -159,7 +158,6 @@ cmake_param_list+=( -D ESPRESSO_BUILD_WITH_CCACHE:BOOL=${with_ccache} -D ESPRESSO_BUILD_WITH_CALIPER:BOOL=${with_caliper} -D ESPRESSO_BUILD_WITH_FPE:BOOL=${with_fpe} - -D ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM:BOOL=${with_shared_memory_parallelism} -D ESPRESSO_BUILD_WITH_HDF5:BOOL=${with_hdf5} -D ESPRESSO_BUILD_WITH_FFTW:BOOL=${with_fftw} -D ESPRESSO_BUILD_WITH_GSL:BOOL=${with_gsl} diff --git a/src/config/features.def b/src/config/features.def index 2ce596ac87..1e6f3ab8c1 100644 --- a/src/config/features.def +++ b/src/config/features.def @@ -102,4 +102,3 @@ WALBERLA external VALGRIND external CALIPER external FPE external -SHARED_MEMORY_PARALLELISM external diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 611efca04f..11e024c159 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -76,17 +76,12 @@ install(TARGETS espresso_core target_link_libraries( espresso_core - PRIVATE - espresso::config - espresso::utils::mpi - espresso::shapes - espresso::compiler_flags - $<$:OpenMP::OpenMP_CXX> - $<$:Kokkos::kokkos> - $<$:Cabana::Core> - $<$:espresso::walberla> - $<$:nlopt> - $<$:GSL::gsl> + PRIVATE espresso::config espresso::utils::mpi espresso::shapes + espresso::compiler_flags OpenMP::OpenMP_CXX Kokkos::kokkos + Cabana::Core + $<$:espresso::walberla> + $<$:nlopt> + $<$:GSL::gsl> PUBLIC espresso::utils MPI::MPI_CXX Random123 espresso::particle_observables Boost::serialization Boost::mpi espresso::instrumentation) diff --git a/src/core/bond_breakage/bond_breakage.cpp b/src/core/bond_breakage/bond_breakage.cpp index 445bcb418f..296eec5282 100644 --- a/src/core/bond_breakage/bond_breakage.cpp +++ b/src/core/bond_breakage/bond_breakage.cpp @@ -17,8 +17,10 @@ * along with this program. If not, see . */ -#include "bond_breakage/bond_breakage.hpp" +#include + #include "bond_breakage/actions.hpp" +#include "bond_breakage/bond_breakage.hpp" #include "cell_system/CellStructure.hpp" #include "communication.hpp" @@ -54,9 +56,7 @@ void BondBreakage::queue_breakage(int particle_id, BondPartners const &bond_partners, int bond_type) { { -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM std::lock_guard lock(queue_mtx); -#endif m_queue.emplace_back(QueueEntry{particle_id, bond_partners, bond_type}); } } diff --git a/src/core/bond_breakage/bond_breakage.hpp b/src/core/bond_breakage/bond_breakage.hpp index e679c8f776..dfdb7a5e93 100644 --- a/src/core/bond_breakage/bond_breakage.hpp +++ b/src/core/bond_breakage/bond_breakage.hpp @@ -19,6 +19,8 @@ #pragma once +#include + #include "system/System.hpp" #include @@ -106,9 +108,7 @@ class BondBreakage { } private: -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM std::mutex queue_mtx; -#endif void process_queue_impl(System::System &system); diff --git a/src/core/bond_forces_kokkos.hpp b/src/core/bond_forces_kokkos.hpp index e6bf57f55a..a1be170841 100644 --- a/src/core/bond_forces_kokkos.hpp +++ b/src/core/bond_forces_kokkos.hpp @@ -21,8 +21,6 @@ #include -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM - #include "aosoa_pack.hpp" #include "cell_system/LocalBondState.hpp" #include "forces_inline.hpp" @@ -277,5 +275,3 @@ struct DihedralBondsKernel { } } }; - -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM diff --git a/src/core/cell_system/CellStructure.cpp b/src/core/cell_system/CellStructure.cpp index 61623b6fbf..16d8ede17a 100644 --- a/src/core/cell_system/CellStructure.cpp +++ b/src/core/cell_system/CellStructure.cpp @@ -411,7 +411,6 @@ int CellStructure::get_max_local_particle_id() const { return (it != m_particle_index.rend()) ? (*it)->id() : -1; } -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM int CellStructure::get_local_pair_bond_numbers() const { return m_bond_state->pair_count; } @@ -433,7 +432,6 @@ void CellStructure::add_new_bond(int bond_id, } void CellStructure::rebuild_bond_list() { m_bond_state->rebuild(); } #endif // ESPRESSO_COLLISION_DETECTION -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM void CellStructure::remove_all_particles() { for (auto cell : decomposition().local_cells()) { @@ -441,9 +439,7 @@ void CellStructure::remove_all_particles() { } m_particle_index.clear(); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM clear_bond_properties(); -#endif } /* Map the data parts flags from cells to those used internally diff --git a/src/core/cell_system/LocalBondState.cpp b/src/core/cell_system/LocalBondState.cpp index e82f2cd139..0eec20a411 100644 --- a/src/core/cell_system/LocalBondState.cpp +++ b/src/core/cell_system/LocalBondState.cpp @@ -21,8 +21,6 @@ #include "LocalBondState.hpp" -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM - void LocalBondState::allocate() { if (pair_list.is_allocated()) { Kokkos::realloc(Kokkos::view_alloc(Kokkos::WithoutInitializing), pair_list, @@ -175,4 +173,3 @@ void LocalBondState::rebuild() { clear_new_bonds(); } #endif // ESPRESSO_COLLISION_DETECTION -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM diff --git a/src/core/cell_system/LocalBondState.hpp b/src/core/cell_system/LocalBondState.hpp index dc42f33238..4dc09c2594 100644 --- a/src/core/cell_system/LocalBondState.hpp +++ b/src/core/cell_system/LocalBondState.hpp @@ -21,9 +21,8 @@ #include -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM - #include + #include struct LocalBondState { @@ -81,5 +80,3 @@ struct LocalBondState { void rebuild(); #endif }; - -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM diff --git a/src/core/collision_detection/utils.hpp b/src/core/collision_detection/utils.hpp index 43128982e7..b15f9a44cd 100644 --- a/src/core/collision_detection/utils.hpp +++ b/src/core/collision_detection/utils.hpp @@ -94,9 +94,7 @@ inline void add_bind_centers(std::vector &collision_queue, // Once we change bond storage, some syncing has to be done. assert(use_one_sided_bond_storage); ::add_bond(system, bond_id, {c.first, c.second}); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM system.cell_structure->add_new_bond(bond_id, {c.first, c.second}); -#endif } } diff --git a/src/core/forces_inline.hpp b/src/core/forces_inline.hpp index 177df224f8..a7b10a3a07 100644 --- a/src/core/forces_inline.hpp +++ b/src/core/forces_inline.hpp @@ -329,9 +329,7 @@ inline void add_non_bonded_pair_force( * @param[in] dx Vector between @p p1 and @p p2. * @param[in] kernel Coulomb force kernel. */ -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM ESPRESSO_ATTR_ALWAYS_INLINE -#endif inline std::optional calc_bond_pair_force( Bonded_IA_Parameters const &iaparams, Utils::Vector3d const &dx, double const q1q2, @@ -408,9 +406,7 @@ inline bool add_bonded_two_body_force( return true; } -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM ESPRESSO_ATTR_ALWAYS_INLINE -#endif inline std::optional< std::tuple> calc_bonded_three_body_force(Bonded_IA_Parameters const &iaparams, @@ -458,9 +454,7 @@ inline bool add_bonded_three_body_force(Bonded_IA_Parameters const &iaparams, return true; } -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM ESPRESSO_ATTR_ALWAYS_INLINE -#endif inline std::optional> calc_bonded_four_body_force( diff --git a/src/core/integrate.cpp b/src/core/integrate.cpp index b1854f911f..3d886c42ec 100644 --- a/src/core/integrate.cpp +++ b/src/core/integrate.cpp @@ -808,13 +808,9 @@ int System::System::integrate(int n_steps, int reuse_forces) { #endif #ifdef ESPRESSO_COLLISION_DETECTION -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM cell_structure->clear_new_bonds(); -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM collision_detection->handle_collisions(); -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM cell_structure->rebuild_bond_list(); -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM #endif bond_breakage->process_queue(*this); } diff --git a/src/core/p3m/CMakeLists.txt b/src/core/p3m/CMakeLists.txt index 6b55c2ad6f..4d121287dd 100644 --- a/src/core/p3m/CMakeLists.txt +++ b/src/core/p3m/CMakeLists.txt @@ -33,19 +33,10 @@ if(ESPRESSO_BUILD_WITH_FFTW) ) target_link_libraries( espresso_p3m - PRIVATE - espresso::compiler_flags - espresso::p3m::compiler_flags - espresso::instrumentation - espresso::utils - espresso::config - Random123 - Boost::mpi - MPI::MPI_CXX - Heffte::Heffte - $<$:OpenMP::OpenMP_CXX> - $<$:Kokkos::kokkos> - $<$:Cabana::Core>) + PRIVATE espresso::compiler_flags espresso::p3m::compiler_flags + espresso::instrumentation espresso::utils espresso::config + Random123 Boost::mpi MPI::MPI_CXX Heffte::Heffte OpenMP::OpenMP_CXX + Kokkos::kokkos Cabana::Core) set_target_properties(espresso_p3m PROPERTIES POSITION_INDEPENDENT_CODE ON) target_link_libraries(espresso_core PRIVATE espresso_p3m) target_sources( diff --git a/src/core/unit_tests/CMakeLists.txt b/src/core/unit_tests/CMakeLists.txt index 9030b5aaea..59d9122368 100644 --- a/src/core/unit_tests/CMakeLists.txt +++ b/src/core/unit_tests/CMakeLists.txt @@ -31,16 +31,14 @@ espresso_unit_test_executable( MPI::MPI_CXX) foreach(TEST_NUM_PROC 1 3 4) foreach(TEST_NUM_THREADS 1 2) - if(${ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM} OR ${TEST_NUM_THREADS} - EQUAL 1) - if(${ESPRESSO_TEST_NP} GREATER_EQUAL ${TEST_NUM_PROC}) + if(${TEST_NUM_THREADS} LESS_EQUAL ${ESPRESSO_TEST_NT}) + if(${TEST_NUM_PROC} LESS_EQUAL ${ESPRESSO_TEST_NP}) set(TEST_NAME EspressoSystemStandAlone_test_${TEST_NUM_PROC}_mpi_ranks_${TEST_NUM_THREADS}_omp_threads ) espresso_unit_test_register( - NAME ${TEST_NAME} TARGET EspressoSystemStandAlone_test_matrix DEPENDS - espresso::core Boost::mpi MPI::MPI_CXX NUM_PROC ${TEST_NUM_PROC} - NUM_THREADS ${TEST_NUM_THREADS}) + NAME ${TEST_NAME} TARGET EspressoSystemStandAlone_test_matrix + NUM_PROC ${TEST_NUM_PROC} NUM_THREADS ${TEST_NUM_THREADS}) endif() endif() endforeach() @@ -63,13 +61,10 @@ espresso_unit_test(SRC field_coupling_couplings_test.cpp DEPENDS espresso_unit_test(SRC field_coupling_fields_test.cpp DEPENDS espresso::utils) espresso_unit_test(SRC field_coupling_force_field_test.cpp DEPENDS espresso::utils) -espresso_unit_test( - SRC field_layout_helpers_test.cpp DEPENDS espresso::utils espresso::core - $<$:Kokkos::kokkos>) -espresso_unit_test( - SRC particle_reduction_test.cpp DEPENDS espresso::utils espresso::core - Boost::mpi - $<$:Kokkos::kokkos>) +espresso_unit_test(SRC field_layout_helpers_test.cpp DEPENDS espresso::utils + espresso::core Kokkos::kokkos) +espresso_unit_test(SRC particle_reduction_test.cpp DEPENDS espresso::utils + espresso::core Boost::mpi Kokkos::kokkos) espresso_unit_test(SRC periodic_fold_test.cpp) espresso_unit_test(SRC grid_test.cpp DEPENDS espresso::core) espresso_unit_test(SRC lees_edwards_test.cpp DEPENDS espresso::core) @@ -101,27 +96,24 @@ if(ESPRESSO_BUILD_WITH_FFTW) NAME p3m_test_matrix SRC p3m_test.cpp DEPENDS espresso::core Boost::mpi MPI::MPI_CXX $<$:espresso::p3m::compiler_flags> - $<$:Heffte::Heffte> - $<$:Cabana::Core> - $<$:Kokkos::kokkos>) + $<$:Heffte::Heffte> Cabana::Core + Kokkos::kokkos) foreach(TEST_NUM_PROC 1 3 4) foreach(TEST_NUM_THREADS 1 2) - if(${ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM} OR ${TEST_NUM_THREADS} - EQUAL 1) - if(${ESPRESSO_TEST_NP} GREATER_EQUAL ${TEST_NUM_PROC}) + if(${TEST_NUM_THREADS} LESS_EQUAL ${ESPRESSO_TEST_NT}) + if(${TEST_NUM_PROC} LESS_EQUAL ${ESPRESSO_TEST_NP}) set(TEST_NAME p3m_test_${TEST_NUM_PROC}_mpi_ranks_${TEST_NUM_THREADS}_omp_threads ) espresso_unit_test_register( - NAME ${TEST_NAME} TARGET p3m_test_matrix DEPENDS MPI::MPI_CXX - NUM_PROC ${TEST_NUM_PROC} NUM_THREADS ${TEST_NUM_THREADS}) + NAME ${TEST_NAME} TARGET p3m_test_matrix NUM_PROC ${TEST_NUM_PROC} + NUM_THREADS ${TEST_NUM_THREADS}) endif() endif() endforeach() endforeach() - espresso_unit_test( - SRC fft_test.cpp DEPENDS espresso::utils espresso::core - $<$:Kokkos::kokkos>) + espresso_unit_test(SRC fft_test.cpp DEPENDS espresso::utils espresso::core + Kokkos::kokkos) espresso_unit_test(SRC math_test.cpp DEPENDS espresso::utils espresso::core) endif() diff --git a/src/core/unit_tests/EspressoSystemStandAlone_test.cpp b/src/core/unit_tests/EspressoSystemStandAlone_test.cpp index 3e9ace6fa8..a1d467c138 100644 --- a/src/core/unit_tests/EspressoSystemStandAlone_test.cpp +++ b/src/core/unit_tests/EspressoSystemStandAlone_test.cpp @@ -551,7 +551,6 @@ BOOST_FIXTURE_TEST_CASE(espresso_system_stand_alone, ParticleFactory) { // check bond counting { -#ifdef ESPRESSO_SHARED_MEMORY_PARALLELISM auto &cs = *system.cell_structure; auto init_n_pairs = 0; auto init_n_angles = 0; @@ -584,7 +583,6 @@ BOOST_FIXTURE_TEST_CASE(espresso_system_stand_alone, ParticleFactory) { BOOST_CHECK_EQUAL(cs.get_local_angle_bond_numbers(), init_n_angles); BOOST_CHECK_EQUAL(cs.get_local_dihedral_bond_numbers(), init_n_dihes); #endif // ESPRESSO_COLLISION_DETECTION -#endif // ESPRESSO_SHARED_MEMORY_PARALLELISM } // check exceptions from sanity checks diff --git a/src/python/espressomd/CMakeLists.txt b/src/python/espressomd/CMakeLists.txt index 378f8895f6..0c5fac514f 100644 --- a/src/python/espressomd/CMakeLists.txt +++ b/src/python/espressomd/CMakeLists.txt @@ -116,16 +116,14 @@ foreach(cython_file ${cython_SRC}) ${target} PROPERTIES SUFFIX ".so" LINK_FLAGS "-undefined dynamic_lookup") endif() set_target_properties(${target} PROPERTIES CXX_CLANG_TIDY "") - if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) - if(CMAKE_CXX_COMPILER_ID STREQUAL CrayClang AND target STREQUAL - espressomd__init - AND DEFINED OpenMP_craymp_LIBRARY AND EXISTS ${OpenMP_craymp_LIBRARY}) - # the Cray OpenMP library must be loaded before dependent Cray libraries, - # including those imported from target OpenMP::OpenMP_CXX, so as to avoid - # "ImportError: cannot allocate memory in static TLS block" - target_link_libraries(${target} PRIVATE ${OpenMP_craymp_LIBRARY} - OpenMP::OpenMP_CXX) - endif() + if(CMAKE_CXX_COMPILER_ID STREQUAL CrayClang AND target STREQUAL + espressomd__init + AND DEFINED OpenMP_craymp_LIBRARY AND EXISTS ${OpenMP_craymp_LIBRARY}) + # the Cray OpenMP library must be loaded before dependent Cray libraries, + # including those imported from target OpenMP::OpenMP_CXX, so as to avoid + # "ImportError: cannot allocate memory in static TLS block" + target_link_libraries(${target} PRIVATE ${OpenMP_craymp_LIBRARY} + OpenMP::OpenMP_CXX) endif() target_link_libraries(${target} PRIVATE espresso::config espresso::core espresso::script_interface) diff --git a/src/walberla_bridge/CMakeLists.txt b/src/walberla_bridge/CMakeLists.txt index a12950428b..76947b5916 100644 --- a/src/walberla_bridge/CMakeLists.txt +++ b/src/walberla_bridge/CMakeLists.txt @@ -126,25 +126,18 @@ espresso_configure_walberla_target(espresso_walberla) espresso_configure_walberla_target(espresso_walberla_codegen) target_link_libraries( - espresso_walberla - PUBLIC MPI::MPI_CXX espresso::utils - PRIVATE - espresso::compiler_flags - espresso::walberla::compiler_flags - espresso::walberla_codegen - $<$:Heffte::Heffte> - $<$:espresso::avx_flags> - $<$:OpenMP::OpenMP_CXX> -) + espresso_walberla PUBLIC MPI::MPI_CXX espresso::utils + PRIVATE espresso::compiler_flags espresso::walberla::compiler_flags + espresso::walberla_codegen + $<$:Heffte::Heffte> + $<$:espresso::avx_flags> + OpenMP::OpenMP_CXX) target_link_libraries( espresso_walberla_codegen - PRIVATE - espresso::compiler_flags - espresso::walberla::compiler_flags - espresso::walberla_codegen::compiler_flags - $<$:espresso::avx_flags> - $<$:OpenMP::OpenMP_CXX> -) + PRIVATE espresso::compiler_flags espresso::walberla::compiler_flags + espresso::walberla_codegen::compiler_flags + $<$:espresso::avx_flags> + OpenMP::OpenMP_CXX) function(espresso_set_walberla_codegen_kernel_source_properties source_file) # disable sanitizers on kernels due to excessive runtime and VRAM usage diff --git a/src/walberla_bridge/tests/CMakeLists.txt b/src/walberla_bridge/tests/CMakeLists.txt index bfe78425fc..cd0be82d27 100644 --- a/src/walberla_bridge/tests/CMakeLists.txt +++ b/src/walberla_bridge/tests/CMakeLists.txt @@ -35,7 +35,7 @@ function(ESPRESSO_ADD_TEST) DEPENDS ${TEST_DEPENDS} $<$:espresso::avx_flags> - $<$:OpenMP::OpenMP_CXX> + OpenMP::OpenMP_CXX espresso::walberla espresso::walberla_deps espresso::compiler_flags diff --git a/testsuite/python/CMakeLists.txt b/testsuite/python/CMakeLists.txt index 1b4ac9c704..f038ffc5c9 100644 --- a/testsuite/python/CMakeLists.txt +++ b/testsuite/python/CMakeLists.txt @@ -123,10 +123,7 @@ function(python_test) endif() list(APPEND TEST_LABELS "python_test") - set(TEST_NUM_CORES ${TEST_NUM_PROC}) - if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) - math(EXPR TEST_NUM_CORES "${TEST_NUM_PROC} * ${TEST_NUM_THREADS}") - endif() + math(EXPR TEST_NUM_CORES "${TEST_NUM_PROC} * ${TEST_NUM_THREADS}") # cmake-format: off set_tests_properties( diff --git a/testsuite/python/caliper.py b/testsuite/python/caliper.py index dcba35a737..3e94721faa 100644 --- a/testsuite/python/caliper.py +++ b/testsuite/python/caliper.py @@ -25,39 +25,33 @@ import sys import os -HAS_CABANA = espressomd.has_features(["SHARED_MEMORY_PARALLELISM"]) EXPECTED_LABELS = """ integrate - {update_cabana} + update_cabana_state Initial Force Calculation calculate_forces - {gpu_to} - {update_cabana} + copy_particles_to_GPU + update_cabana_state init_forces_and_thermostat calc_long_range_forces - {short_range} - {gpu_from} + cabana_short_range + copy_forces_from_GPU Integration loop integrator_step_1 resort_particles_if_needed calculate_forces - {gpu_to} - {update_cabana} + copy_particles_to_GPU + update_cabana_state init_forces_and_thermostat calc_long_range_forces - {short_range} - {gpu_from} + cabana_short_range + copy_forces_from_GPU integrator_step_2 calc_energies - {update_cabana} + update_cabana_state short_range_loop -""".format( - update_cabana='update_cabana_state' if HAS_CABANA else '', - short_range='cabana_short_range' if HAS_CABANA else 'serial_short_range', - gpu_to='copy_particles_to_GPU', - gpu_from='copy_forces_from_GPU' -) +""" @utx.skipIfMissingFeatures(["CALIPER"]) diff --git a/testsuite/python/cell_system.py b/testsuite/python/cell_system.py index fb1745b752..f4658ea74e 100644 --- a/testsuite/python/cell_system.py +++ b/testsuite/python/cell_system.py @@ -94,7 +94,7 @@ def test_node_grid_hybrid(self): n_square_types={1}, cutoff_regular=0) self.check_node_grid() - @utx.skipIfMissingFeatures(["WCA", "SHARED_MEMORY_PARALLELISM"]) + @utx.skipIfMissingFeatures(["WCA"]) def test_verlet_list_overflow(self): system = self.system system.part.clear() From 941fb0da976957f73d1f8ded9d0af28a97683013 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Grad?= Date: Thu, 2 Apr 2026 00:39:00 +0200 Subject: [PATCH 03/11] WIP: fix CMake and try new images with hwloc --- .github/workflows/push_pull.yml | 8 ++++---- CMakeLists.txt | 6 +++++- src/core/particle_reduction.hpp | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/push_pull.yml b/.github/workflows/push_pull.yml index 53cb5ec778..0f26f175c2 100644 --- a/.github/workflows/push_pull.yml +++ b/.github/workflows/push_pull.yml @@ -52,7 +52,7 @@ jobs: debian: runs-on: ubuntu-latest container: - image: ghcr.io/espressomd/docker/debian:4d3e236db5ae334ddcc9c3c82aae7637faa5dabc-base-layer + image: ghcr.io/espressomd/docker/debian:d63998a4eb491c48130038b0c5ca07d24b254bc2-base-layer credentials: username: ${{ github.actor }} password: ${{ secrets.github_token }} @@ -93,7 +93,7 @@ jobs: runs-on: ubuntu-latest if: ${{ github.repository == 'espressomd/espresso' }} container: - image: ghcr.io/espressomd/docker/ubuntu-wo-dependencies:4d3e236db5ae334ddcc9c3c82aae7637faa5dabc-base-layer + image: ghcr.io/espressomd/docker/ubuntu-wo-dependencies:d63998a4eb491c48130038b0c5ca07d24b254bc2-base-layer credentials: username: ${{ github.actor }} password: ${{ secrets.github_token }} @@ -135,7 +135,7 @@ jobs: runs-on: ubuntu-latest if: ${{ github.repository == 'espressomd/espresso' }} container: - image: ghcr.io/espressomd/docker/fedora:4d3e236db5ae334ddcc9c3c82aae7637faa5dabc-base-layer + image: ghcr.io/espressomd/docker/fedora:d63998a4eb491c48130038b0c5ca07d24b254bc2-base-layer credentials: username: ${{ github.actor }} password: ${{ secrets.github_token }} @@ -165,7 +165,7 @@ jobs: with_hdf5: 'true' with_fftw: 'true' with_fpe: 'true' - with_gsl: 'false' + with_gsl: 'true' with_scafacos: 'false' with_walberla: 'true' with_walberla_avx: 'false' diff --git a/CMakeLists.txt b/CMakeLists.txt index 49a1316b4b..bc59b285e1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -680,7 +680,6 @@ if(NOT DEFINED Kokkos_FOUND OR NOT ${Kokkos_FOUND}) TARGET kokkos APPEND PROPERTY INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $) - set_target_properties(kokkos PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "") endif() if(NOT EXISTS ${FETCHCONTENT_BASE_DIR}/cabana-src) @@ -697,6 +696,11 @@ if(NOT DEFINED Cabana_FOUND OR NOT ${Cabana_FOUND}) # cmake-format: on set(Cabana_REQUIRE_HEFFTE ${ESPRESSO_BUILD_WITH_FFTW} CACHE BOOL "") FetchContent_MakeAvailable(cabana) + # mark Cabana headers as system headers to disable compiler diagnostics + set_property( + TARGET Core APPEND + PROPERTY INTERFACE_SYSTEM_INCLUDE_DIRECTORIES + $) endif() # We need the parallel hdf5 version! diff --git a/src/core/particle_reduction.hpp b/src/core/particle_reduction.hpp index 5858f00249..223cd5bf70 100644 --- a/src/core/particle_reduction.hpp +++ b/src/core/particle_reduction.hpp @@ -58,7 +58,7 @@ template class KokkosReducer { KokkosReducer(Kernel kernel, ReductionOp reduction_op) : reduction_op(std::move(reduction_op)), kernel(std::move(kernel)) {} KokkosReducer(KokkosReducer const &other) - : reduction_op(other.reduction_op), kernel(other.kernel) {}; + : reduction_op(other.reduction_op), kernel(other.kernel) {} KOKKOS_INLINE_FUNCTION void operator()(std::integral auto const i, value_type &update) const { From dce19bb7d1761820f55b8381c819ac8a4318fc38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Grad?= Date: Thu, 2 Apr 2026 01:37:08 +0200 Subject: [PATCH 04/11] WIP: FPE instrumentation --- testsuite/python/cell_system.py | 2 ++ testsuite/python/dipolar_mdlc_p3m_scafacos_p2nfft.py | 2 ++ testsuite/python/scafacos_dipoles_1d_2d.py | 2 ++ testsuite/python/scafacos_interface.py | 4 ++++ 4 files changed, 10 insertions(+) diff --git a/testsuite/python/cell_system.py b/testsuite/python/cell_system.py index f4658ea74e..3b473b3ede 100644 --- a/testsuite/python/cell_system.py +++ b/testsuite/python/cell_system.py @@ -95,6 +95,8 @@ def test_node_grid_hybrid(self): self.check_node_grid() @utx.skipIfMissingFeatures(["WCA"]) + @ut.skipIf(espressomd.has_features("FPE"), + "cannot run with FPE instrumentation") def test_verlet_list_overflow(self): system = self.system system.part.clear() diff --git a/testsuite/python/dipolar_mdlc_p3m_scafacos_p2nfft.py b/testsuite/python/dipolar_mdlc_p3m_scafacos_p2nfft.py index ddf3e66a97..2cf91de9ae 100644 --- a/testsuite/python/dipolar_mdlc_p3m_scafacos_p2nfft.py +++ b/testsuite/python/dipolar_mdlc_p3m_scafacos_p2nfft.py @@ -33,6 +33,8 @@ @utx.skipIfMissingFeatures(["DIPOLES", "FFTW"]) +@ut.skipIf(espressomd.has_features("FPE"), + "cannot run with FPE instrumentation") class Test(ut.TestCase): """Tests mdlc (2d) as well as dipolar p3m and dipolar p2nfft (3d) against diff --git a/testsuite/python/scafacos_dipoles_1d_2d.py b/testsuite/python/scafacos_dipoles_1d_2d.py index 337342b8ca..bc2590840b 100644 --- a/testsuite/python/scafacos_dipoles_1d_2d.py +++ b/testsuite/python/scafacos_dipoles_1d_2d.py @@ -31,6 +31,8 @@ @utx.skipIfMissingFeatures(["SCAFACOS_DIPOLES"]) +@ut.skipIf(espressomd.has_features("FPE"), + "cannot run with FPE instrumentation") class Scafacos1d2d(ut.TestCase): system = espressomd.System(box_l=[1.0, 1.0, 1.0]) diff --git a/testsuite/python/scafacos_interface.py b/testsuite/python/scafacos_interface.py index 9190d475f6..804b30aea3 100644 --- a/testsuite/python/scafacos_interface.py +++ b/testsuite/python/scafacos_interface.py @@ -74,6 +74,8 @@ def test_magnetostatics_actor_exceptions(self): @utx.skipIfMissingFeatures(["SCAFACOS"]) @utx.skipIfMissingScafacosMethod("p3m") @utx.skipIfMissingScafacosMethod("ewald") + @ut.skipIf(espressomd.has_features("FPE"), + "cannot run with FPE instrumentation") def test_electrostatics_actor_exceptions(self): with self.assertRaisesRegex(ValueError, "Parameter 'prefactor' must be > 0"): espressomd.electrostatics.Scafacos( @@ -218,6 +220,8 @@ def test_tuning_alpha_ewald(self): @utx.skipIfMissingFeatures(["SCAFACOS_DIPOLES"]) @utx.skipIfMissingScafacosMethod("p2nfft") + @ut.skipIf(espressomd.has_features("FPE"), + "cannot run with FPE instrumentation") def test_actor_dipoles(self): system = self.system From ceaffd1409f3295e06590185d482cf523299a454 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Grad?= Date: Thu, 2 Apr 2026 17:59:26 +0200 Subject: [PATCH 05/11] Update images --- .github/workflows/push_pull.yml | 6 +++--- .gitlab-ci.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/push_pull.yml b/.github/workflows/push_pull.yml index 0f26f175c2..96d40f6468 100644 --- a/.github/workflows/push_pull.yml +++ b/.github/workflows/push_pull.yml @@ -52,7 +52,7 @@ jobs: debian: runs-on: ubuntu-latest container: - image: ghcr.io/espressomd/docker/debian:d63998a4eb491c48130038b0c5ca07d24b254bc2-base-layer + image: ghcr.io/espressomd/docker/debian:a55be868226f9b1b313deea773ce01705707a6f1-base-layer credentials: username: ${{ github.actor }} password: ${{ secrets.github_token }} @@ -93,7 +93,7 @@ jobs: runs-on: ubuntu-latest if: ${{ github.repository == 'espressomd/espresso' }} container: - image: ghcr.io/espressomd/docker/ubuntu-wo-dependencies:d63998a4eb491c48130038b0c5ca07d24b254bc2-base-layer + image: ghcr.io/espressomd/docker/ubuntu-wo-dependencies:a55be868226f9b1b313deea773ce01705707a6f1-base-layer credentials: username: ${{ github.actor }} password: ${{ secrets.github_token }} @@ -135,7 +135,7 @@ jobs: runs-on: ubuntu-latest if: ${{ github.repository == 'espressomd/espresso' }} container: - image: ghcr.io/espressomd/docker/fedora:d63998a4eb491c48130038b0c5ca07d24b254bc2-base-layer + image: ghcr.io/espressomd/docker/fedora:a55be868226f9b1b313deea773ce01705707a6f1-base-layer credentials: username: ${{ github.actor }} password: ${{ secrets.github_token }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index baf0146044..4182a4af78 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: ghcr.io/espressomd/docker/ubuntu:4d3e236db5ae334ddcc9c3c82aae7637faa5dabc +image: ghcr.io/espressomd/docker/ubuntu:a55be868226f9b1b313deea773ce01705707a6f1 stages: - prepare @@ -16,7 +16,7 @@ stages: .notification_job_template: ¬ification_job_definition <<: *global_job_definition - image: ghcr.io/espressomd/docker/alpine:4d3e236db5ae334ddcc9c3c82aae7637faa5dabc + image: ghcr.io/espressomd/docker/alpine:a55be868226f9b1b313deea773ce01705707a6f1 variables: GET_SOURCES_ATTEMPTS: 3 dependencies: [] From 908532bcc768ab645063c55a6a1e2dd65f743638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Grad?= Date: Thu, 2 Apr 2026 20:35:50 +0200 Subject: [PATCH 06/11] Rewrite FPE signal handler --- CMakeLists.txt | 17 +++++++ src/config/features.def | 1 + src/core/electrostatics/p3m_gpu_cuda.cu | 21 ++++++-- src/core/integrate.cpp | 10 ++++ .../include/instrumentation/fe_trap.hpp | 50 ++++++++++++++++++- src/instrumentation/src/fe_trap.cpp | 29 +++++++++-- src/instrumentation/tests/fe_trap_test.cpp | 48 ++++++++++++++++++ .../dipolar_mdlc_p3m_scafacos_p2nfft.py | 2 - testsuite/python/scafacos_dipoles_1d_2d.py | 2 - testsuite/python/scafacos_interface.py | 4 -- 10 files changed, 164 insertions(+), 20 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bc59b285e1..a7e408bcf1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -450,6 +450,23 @@ target_compile_options( $<$,$>:-Werror> $<$,$>:--Werror=all-warnings> $<$,$>:-Werror> +) +# cmake-format: on + +# +# Compiler optimizations +# + +# cmake-format: off +target_compile_options( + espresso_compiler_flags + INTERFACE + # prevent loop auto-vectorization when doing so can introduce invalid math; + # for example, when a loop body contains a conditional to skip divisions by + # zero, the conditional can be ignored (speculative execution) to get better + # performance with SIMD types, and the invalid values are masked afterwards + # (but the SIGFPE signal is still sent as a side effect) + $<$,$>:-ftrapping-math> # configurations for NVCC $<$,$>:-g -G> $<$,$>:-Xptxas=-O3 -Xcompiler=-O3 -DNDEBUG> diff --git a/src/config/features.def b/src/config/features.def index 1e6f3ab8c1..2045216378 100644 --- a/src/config/features.def +++ b/src/config/features.def @@ -84,6 +84,7 @@ THOLE requires ELECTROSTATICS SCAFACOS_DIPOLES requires SCAFACOS SCAFACOS_DIPOLES implies DIPOLES SCAFACOS requires ELECTROSTATICS +SCAFACOS requires not FPE /* Debugging */ ADDITIONAL_CHECKS diff --git a/src/core/electrostatics/p3m_gpu_cuda.cu b/src/core/electrostatics/p3m_gpu_cuda.cu index a9411b5057..bfb87563f4 100644 --- a/src/core/electrostatics/p3m_gpu_cuda.cu +++ b/src/core/electrostatics/p3m_gpu_cuda.cu @@ -56,6 +56,8 @@ #include "p3m/math.hpp" #include "system/System.hpp" +#include + #include #include #include @@ -613,11 +615,20 @@ void p3m_gpu_init(std::shared_ptr &data, int cao, cuda_safe_mem(cudaMalloc((void **)&(p3m_gpu_data.G_hat), cmesh_size * sizeof(REAL_TYPE))); - if (cufftPlan3d(&(data->p3m_fft.forw_plan), mesh[0], mesh[1], mesh[2], - FFT_PLAN_FORW_FLAG) != CUFFT_SUCCESS or - cufftPlan3d(&(data->p3m_fft.back_plan), mesh[0], mesh[1], mesh[2], - FFT_PLAN_BACK_FLAG) != CUFFT_SUCCESS) { - throw std::runtime_error("Unable to create fft plan"); + { +#ifdef ESPRESSO_FPE + // starting with CUDA 12.0, cuFFT builds device kernels using CUDA-JIT + // (https://docs.nvidia.com/cuda/archive/13.1.1/cufft/#plan-initialization-time) + // at runtime; this operation is not guaranteed to succeed for all mesh + // sizes, and in rare cases, it can send the SIGFPE signal + auto const trap_pause = fe_trap::make_shared_pause_scoped(); +#endif + if (cufftPlan3d(&(data->p3m_fft.forw_plan), mesh[0], mesh[1], mesh[2], + FFT_PLAN_FORW_FLAG) != CUFFT_SUCCESS or + cufftPlan3d(&(data->p3m_fft.back_plan), mesh[0], mesh[1], mesh[2], + FFT_PLAN_BACK_FLAG) != CUFFT_SUCCESS) { + throw std::runtime_error("Unable to create fft plan"); + } } } diff --git a/src/core/integrate.cpp b/src/core/integrate.cpp index 3d886c42ec..4a3c134efd 100644 --- a/src/core/integrate.cpp +++ b/src/core/integrate.cpp @@ -62,6 +62,8 @@ #include "virtual_sites/lb_tracers.hpp" #include "virtual_sites/relative.hpp" +#include + #include #ifdef ESPRESSO_CALIPER @@ -752,6 +754,11 @@ int System::System::integrate(int n_steps, int reuse_forces) { lb.ghost_communication_vel(); #ifdef ESPRESSO_CALIPER CALI_MARK_END("lb_propagation"); +#endif +#ifdef ESPRESSO_FPE + auto const trap_pause = fe_trap::make_shared_pause_scoped(); +#endif +#ifdef ESPRESSO_CALIPER CALI_MARK_BEGIN("ek_propagation"); #endif ek.propagate(); @@ -777,6 +784,9 @@ int System::System::integrate(int n_steps, int reuse_forces) { propagation.ek_skipped_md_steps += 1; if (propagation.ek_skipped_md_steps >= md_steps_per_ek_step) { propagation.ek_skipped_md_steps = 0; +#ifdef ESPRESSO_FPE + auto const trap_pause = fe_trap::make_shared_pause_scoped(); +#endif #ifdef ESPRESSO_CALIPER CALI_MARK_BEGIN("ek_propagation"); #endif diff --git a/src/instrumentation/include/instrumentation/fe_trap.hpp b/src/instrumentation/include/instrumentation/fe_trap.hpp index d3bc9f7328..47ecc42d93 100644 --- a/src/instrumentation/include/instrumentation/fe_trap.hpp +++ b/src/instrumentation/include/instrumentation/fe_trap.hpp @@ -36,6 +36,7 @@ * the duration of a scoped block. Exception traps are set when the object * is created; when getting out-of-scope, either normally or during stack * unwinding, the exception traps are automatically reset. + * This idiom is called "scope-based resource management". * * Please note "exception" and "exception handling" have a specific meaning * in this context and are completely unrelated to C++ exceptions. @@ -91,6 +92,7 @@ class fe_trap { }; static global_state_params global_state; + /** @brief Scope-based handle to manage an exception trap lifetime. */ struct scoped_instance { explicit scoped_instance(std::shared_ptr ptr) : m_resource{std::move(ptr)} {} @@ -99,12 +101,29 @@ class fe_trap { scoped_instance &operator=(scoped_instance const &) = delete; scoped_instance &operator=(scoped_instance &&) noexcept = default; bool is_unique() const { return m_resource->is_unique(); } + bool is_active() const { return m_resource->is_active(); } int get_flags() const { return m_resource->get_flags(); } + auto &get_trap() { return *m_resource; } + auto const &get_trap() const { return *m_resource; } + bool operator==(scoped_instance const &) const = default; private: std::shared_ptr m_resource; }; + /** @brief Scope-based handle to temporarily disable an exception trap. */ + struct scoped_pause { + std::weak_ptr m_resource; + explicit scoped_pause(std::shared_ptr ptr) : m_resource{ptr} { + ptr->deactivate(); + } + ~scoped_pause() { + if (auto const ptr = m_resource.lock()) { + ptr->activate(); + } + } + }; + struct deleter { void operator()(fe_trap *ptr) { delete ptr; } }; @@ -112,9 +131,15 @@ class fe_trap { int m_flags; bool m_unique; + bool m_active; // this flag is not strictly equivalent to `not m_pause` + std::weak_ptr m_pause; - fe_trap(std::optional excepts, bool unique); - ~fe_trap(); + fe_trap(std::optional excepts, bool unique) + : m_flags{parse_excepts(excepts)}, m_unique{unique}, m_active{false}, + m_pause{} { + activate(); + } + ~fe_trap() { deactivate(); } static int parse_excepts(std::optional excepts); @@ -127,6 +152,8 @@ class fe_trap { int get_flags() const { return m_flags; } /** @brief Check if this handle is a unique handle. */ bool is_unique() const { return m_unique; } + /** @brief Check if this handle has a currrently active trap. */ + bool is_active() const { return m_active; } /** * @brief Generate a unique trap with the lifetime of the current scope. @@ -142,6 +169,25 @@ class fe_trap { */ static scoped_instance make_shared_scoped(std::optional excepts = std::nullopt); + + /** + * @brief Generate a shared handle to temporarily disable any currently + * active exception trap for the lifetime of the current scope. + */ + static std::optional> + make_shared_pause_scoped(); + + /** @brief Manually activate the exception trap. */ + void activate(); + + /** + * @brief Manually deactivate the exception trap. + * Useful when calling a third-party library that is known to send signals. + * This should only be used in exceptional cases + * (@ref make_shared_pause_scoped provides a scope-based alternative). + * Call @ref activate_trap() to re-activate the exception trap. + */ + void deactivate(); }; #endif // ESPRESSO_FPE diff --git a/src/instrumentation/src/fe_trap.cpp b/src/instrumentation/src/fe_trap.cpp index dea52a6fd5..14882f197d 100644 --- a/src/instrumentation/src/fe_trap.cpp +++ b/src/instrumentation/src/fe_trap.cpp @@ -39,10 +39,12 @@ fe_trap::global_state_params fe_trap::global_state{{}, {}}; -fe_trap::fe_trap(std::optional excepts, bool unique) { +void fe_trap::activate() { + if (m_active) { + return; + } #if defined(ESPRESSO_FPE_USING_GLIBC_X86_64) { - m_flags = parse_excepts(excepts); [[maybe_unused]] auto const status = feenableexcept(m_flags); // note: status should be 0 since we use the singleton pattern assert(status == 0); @@ -50,7 +52,6 @@ fe_trap::fe_trap(std::optional excepts, bool unique) { #elif defined(ESPRESSO_FPE_USING_APPLE_ARM_64) { using fpcr_t = decltype(std::fenv_t::__fpcr); - m_flags = parse_excepts(excepts); std::fenv_t env; { [[maybe_unused]] auto const status = std::fegetenv(&env); @@ -65,10 +66,13 @@ fe_trap::fe_trap(std::optional excepts, bool unique) { #else #error "FE not supported" #endif - m_unique = unique; + m_active = true; } -fe_trap::~fe_trap() { +void fe_trap::deactivate() { + if (not m_active) { + return; + } #if defined(ESPRESSO_FPE_USING_GLIBC_X86_64) { [[maybe_unused]] auto const status = fedisableexcept(m_flags); @@ -94,6 +98,7 @@ fe_trap::~fe_trap() { #else #error "FE not supported" #endif + m_active = false; } int fe_trap::parse_excepts(std::optional excepts) { @@ -136,4 +141,18 @@ fe_trap::make_shared_scoped(std::optional excepts) { return fe_trap::scoped_instance(watched); } +std::optional> +fe_trap::make_shared_pause_scoped() { + std::lock_guard lock(fe_trap::global_state.mutex); + if (auto watched = fe_trap::global_state.observer.lock()) { + if (auto pause = watched->m_pause.lock()) { + return pause; + } + auto pause = std::make_shared(watched); + watched->m_pause = pause; + return pause; + } + return std::nullopt; +} + #endif // ESPRESSO_FPE diff --git a/src/instrumentation/tests/fe_trap_test.cpp b/src/instrumentation/tests/fe_trap_test.cpp index 66994cde9c..1302a72686 100644 --- a/src/instrumentation/tests/fe_trap_test.cpp +++ b/src/instrumentation/tests/fe_trap_test.cpp @@ -100,6 +100,7 @@ BOOST_AUTO_TEST_CASE(trap_by_signal) { { auto const trap = fe_trap::make_unique_scoped(); BOOST_REQUIRE(trap.is_unique()); + BOOST_REQUIRE(trap.is_active()); value = 0.; while (sigsetjmp(::jmp_env, 1) == 0) { // LCOV_EXCL_START @@ -117,6 +118,7 @@ BOOST_AUTO_TEST_CASE(trap_by_signal) { { auto const trap = fe_trap::make_shared_scoped(); BOOST_REQUIRE(not trap.is_unique()); + BOOST_REQUIRE(trap.is_active()); value = 0.; while (sigsetjmp(::jmp_env, 1) == 0) { // LCOV_EXCL_START @@ -136,6 +138,8 @@ BOOST_AUTO_TEST_CASE(trap_by_signal) { { auto const trap2 = fe_trap::make_shared_scoped(FE_UNDERFLOW); BOOST_REQUIRE_EQUAL(trap1.get_flags(), trap2.get_flags()); + BOOST_REQUIRE(trap1 == trap2); + BOOST_REQUIRE(&trap1.get_trap() == &trap2.get_trap()); value = 0.; while (sigsetjmp(::jmp_env, 1) == 0) { // LCOV_EXCL_START @@ -151,6 +155,50 @@ BOOST_AUTO_TEST_CASE(trap_by_signal) { ::last_signal_code = 0; } } + { + auto trap = fe_trap::make_unique_scoped(); + BOOST_REQUIRE(trap.is_unique()); + BOOST_REQUIRE(trap.is_active()); + { + // temporarily deactivate trap + auto trap_pause = fe_trap::make_shared_pause_scoped(); + BOOST_REQUIRE(not trap.is_active()); + // without instrumentation, abnormal operations are allowed + value = 1. / bad_denominator; + value = std::exp(bad_exponent); + } + BOOST_REQUIRE(trap.is_active()); + { + // manually deactivate trap + trap.get_trap().deactivate(); + BOOST_REQUIRE(not trap.is_active()); + // without instrumentation, abnormal operations are allowed + value = 1. / bad_denominator; + value = std::exp(bad_exponent); + // deactivating twice is safe (no-op) + trap.get_trap().deactivate(); + BOOST_REQUIRE(!trap.is_active()); + // manually reactivate trap + trap.get_trap().activate(); + BOOST_REQUIRE(trap.is_active()); + // reactivating twice is safe (no-op) + trap.get_trap().activate(); + BOOST_REQUIRE(trap.is_active()); + } + value = 0.; + while (sigsetjmp(::jmp_env, 1) == 0) { + // LCOV_EXCL_START + value = 2.; + value = 0. / bad_denominator; + // LCOV_EXCL_STOP + } + BOOST_CHECK_EQUAL(::last_signal_status, SIGFPE); + BOOST_CHECK_EQUAL(::last_signal_code, FPE_FLTINV); + BOOST_REQUIRE(not std::isnan(value)); + BOOST_REQUIRE_EQUAL(value, 2.); + ::last_signal_status = 0; + ::last_signal_code = 0; + } // reset default signal handler set_sigaction(SIGFPE, &old_action, nullptr); diff --git a/testsuite/python/dipolar_mdlc_p3m_scafacos_p2nfft.py b/testsuite/python/dipolar_mdlc_p3m_scafacos_p2nfft.py index 2cf91de9ae..ddf3e66a97 100644 --- a/testsuite/python/dipolar_mdlc_p3m_scafacos_p2nfft.py +++ b/testsuite/python/dipolar_mdlc_p3m_scafacos_p2nfft.py @@ -33,8 +33,6 @@ @utx.skipIfMissingFeatures(["DIPOLES", "FFTW"]) -@ut.skipIf(espressomd.has_features("FPE"), - "cannot run with FPE instrumentation") class Test(ut.TestCase): """Tests mdlc (2d) as well as dipolar p3m and dipolar p2nfft (3d) against diff --git a/testsuite/python/scafacos_dipoles_1d_2d.py b/testsuite/python/scafacos_dipoles_1d_2d.py index bc2590840b..337342b8ca 100644 --- a/testsuite/python/scafacos_dipoles_1d_2d.py +++ b/testsuite/python/scafacos_dipoles_1d_2d.py @@ -31,8 +31,6 @@ @utx.skipIfMissingFeatures(["SCAFACOS_DIPOLES"]) -@ut.skipIf(espressomd.has_features("FPE"), - "cannot run with FPE instrumentation") class Scafacos1d2d(ut.TestCase): system = espressomd.System(box_l=[1.0, 1.0, 1.0]) diff --git a/testsuite/python/scafacos_interface.py b/testsuite/python/scafacos_interface.py index 804b30aea3..9190d475f6 100644 --- a/testsuite/python/scafacos_interface.py +++ b/testsuite/python/scafacos_interface.py @@ -74,8 +74,6 @@ def test_magnetostatics_actor_exceptions(self): @utx.skipIfMissingFeatures(["SCAFACOS"]) @utx.skipIfMissingScafacosMethod("p3m") @utx.skipIfMissingScafacosMethod("ewald") - @ut.skipIf(espressomd.has_features("FPE"), - "cannot run with FPE instrumentation") def test_electrostatics_actor_exceptions(self): with self.assertRaisesRegex(ValueError, "Parameter 'prefactor' must be > 0"): espressomd.electrostatics.Scafacos( @@ -220,8 +218,6 @@ def test_tuning_alpha_ewald(self): @utx.skipIfMissingFeatures(["SCAFACOS_DIPOLES"]) @utx.skipIfMissingScafacosMethod("p2nfft") - @ut.skipIf(espressomd.has_features("FPE"), - "cannot run with FPE instrumentation") def test_actor_dipoles(self): system = self.system From 5c44fd042929078a4fd4b74221d79f3d3633cea3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Grad?= Date: Fri, 10 Apr 2026 14:06:37 +0200 Subject: [PATCH 07/11] Improve tutorials Improve sampling in the Grand-Canonical Monte Carlo tutorial. Cite the relevant literature. --- doc/bibliography.bib | 24 +++++++++++++++++++ doc/sphinx/introduction.rst | 3 ++- .../CMakeLists.txt | 2 +- .../grand_canonical_monte_carlo.ipynb | 19 +++++++++++---- doc/tutorials/mlip-water/01_TIP4P_water.ipynb | 2 +- .../test_grand_canonical_monte_carlo.py | 2 +- 6 files changed, 43 insertions(+), 9 deletions(-) diff --git a/doc/bibliography.bib b/doc/bibliography.bib index ff3960d997..e147898ecd 100644 --- a/doc/bibliography.bib +++ b/doc/bibliography.bib @@ -916,6 +916,30 @@ @Article{landsgesell17b doi = {10.1140/epjst/e2016-60324-3}, } +@Article{landsgesell20b, + author = {Landsgesell, Jonas and Hebbeker, Pascal and Rud, Oleg and Lunkad, Raju and Ko{\v{s}}ovan, Peter and Holm, Christian}, + title = {{G}rand-reaction method for simulations of ionization equilibria coupled to ion partitioning}, + journal = {Macromolecules}, + year = {2020}, + volume = {53}, + number = {8}, + pages = {3007--3020}, + doi = {10.1021/acs.macromol.0c00260}, + related = {landsgesell20b-err}, + relatedstring = {Corrected in}, +} + +@Article{landsgesell20b-err, + author = {Beyer, David and Landsgesell, Jonas and Hebbeker, Pascal and Rud, Oleg and Lunkad, Raju and Ko{\v{s}}ovan, Peter and Holm, Christian}, + title = {Correction to ``{G}rand-reaction method for simulations of ionization equilibria coupled to ion partitioning''}, + journal = {Macromolecules}, + year = {2022}, + volume = {55}, + number = {3}, + pages = {1088}, + doi = {10.1021/acs.macromol.1c02672}, +} + @Article{lees72a, author = {Lees, A. W. and Edwards, S. F.}, title = {The computer study of transport processes under extreme conditions}, diff --git a/doc/sphinx/introduction.rst b/doc/sphinx/introduction.rst index 21d42e2674..0a543376c7 100644 --- a/doc/sphinx/introduction.rst +++ b/doc/sphinx/introduction.rst @@ -319,7 +319,8 @@ The following tutorials are available: * :file:`active_matter`: Modelling of self-propelling particles. * :file:`electrokinetics`: Modelling electrokinetics together with hydrodynamic interactions. * :file:`constant_pH`: Modelling the titration of a weak acid using the constant pH method -* :file:`widom_insertion`: Measuring the excess chemical potential of a salt solution using the Widom particle insertion method +* :file:`widom_insertion`: Measuring the excess chemical potential of a salt solution using the Widom particle insertion method :cite:`widom63a` +* :file:`grand_canonical_monte_carlo`: Simulating in the Grand-canonical ensemble with the Grand-canonical Monte Carlo method (GCMC) :cite:`landsgesell20b` * :file:`mlip`: Atomistic simulations using machine-learned interatomic potentials and the MACE-MP-0 model :cite:`batatia22a,batatia25a` * :file:`mlip-water`: Modelling water using the TIP4P model :cite:`abascal05a` and machine-learning interatomic potentials with Apax :cite:`schafer25a` diff --git a/doc/tutorials/grand_canonical_monte_carlo/CMakeLists.txt b/doc/tutorials/grand_canonical_monte_carlo/CMakeLists.txt index 734806cb2c..47db86c43f 100644 --- a/doc/tutorials/grand_canonical_monte_carlo/CMakeLists.txt +++ b/doc/tutorials/grand_canonical_monte_carlo/CMakeLists.txt @@ -20,5 +20,5 @@ espresso_add_tutorial( TARGET tutorial_grand_canonical_monte_carlo FILE "grand_canonical_monte_carlo.ipynb" HTML_RUN VAR_SUBST - "\"p3m_params={'mesh':10,'cao':6,'r_cut':8.22}\"" DEPENDS + "\"p3m_params={'mesh':20,'cao':4,'r_cut':5.05}\"" DEPENDS figures/schematic.svg) diff --git a/doc/tutorials/grand_canonical_monte_carlo/grand_canonical_monte_carlo.ipynb b/doc/tutorials/grand_canonical_monte_carlo/grand_canonical_monte_carlo.ipynb index 2e28afdc61..d985e362de 100644 --- a/doc/tutorials/grand_canonical_monte_carlo/grand_canonical_monte_carlo.ipynb +++ b/doc/tutorials/grand_canonical_monte_carlo/grand_canonical_monte_carlo.ipynb @@ -22,9 +22,9 @@ "## Introduction\n", "Often, in soft-matter physics and chemistry we are interested in systems where the particle numbers are not fixed. This is for instance the case in systems where chemical reactions occur or when particles can be exchanged with a reservoir. The case of changing particle numbers is best described by the grand canonical ensemble.\n", "\n", - "Canonical Monte-Carlo or molecular dynamics simulations are not suitable for this task, since they conserve the particle numbers. However, the Metropolis Monte Carlo method can be generalized to the Grand-Canonical ensemble in a straightforward manner, yielding the so-called Grand-Canonical Monte Carlo method (GCMC), which is introduced in this tutorial.\n", + "Canonical Monte-Carlo or molecular dynamics simulations are not suitable for this task, since they conserve the particle numbers. However, the Metropolis Monte Carlo method can be generalized to the Grand-Canonical ensemble in a straightforward manner, yielding the so-called Grand-Canonical Monte Carlo method (GCMC) [4], which is introduced in this tutorial.\n", "\n", - "This tutorial builds on the tutorial [Widom insertion](https://espressomd.github.io/tutorials/widom_insertion/widom_insertion.html), in which you measured the chemical potential of a monovalent salt solution by using a combination of MD and MC techniques. In this part you will make use of the results (i. e. the chemical potentials) of the previous tutorial to simulate the partitioning of salt ions between a reservoir and a polymer solution. Therefore we introduce and use the grand-canonical Monte Carlo method." + "This tutorial builds on the tutorial [Widom insertion](https://espressomd.github.io/tutorials/widom_insertion/widom_insertion.html), in which you measured the chemical potential of a monovalent salt solution by using a combination of MD and MC techniques. In this part you will make use of the results (i.e. the chemical potentials) of the previous tutorial to simulate the partitioning of salt ions between a reservoir and a polymer solution. Therefore we introduce and use the Grand-canonical Monte Carlo method." ] }, { @@ -451,7 +451,7 @@ "C_MONOMERS_SIM = c_monomer.to(\"molecules/reduced_length^3\")\n", "\n", "##### Number of monomers\n", - "N_CHAINS = 10\n", + "N_CHAINS = 12\n", "N_MONOMERS_PER_CHAIN = 5\n", "\n", "##### System size\n", @@ -736,6 +736,14 @@ "plt.show()" ] }, + { + "cell_type": "markdown", + "id": "53a09644-c7c7-487a-88f6-b992a98859bc", + "metadata": {}, + "source": [ + "High variance is to be expected at concentration ratios above $10^1$ due to finite-size effects (see Fig. 2a in [4])." + ] + }, { "cell_type": "markdown", "id": "7618c1dc-525a-4ed3-b05e-098628fe5201", @@ -773,7 +781,7 @@ "system.thermostat.set_langevin(kT=KT, gamma=GAMMA, seed=langevin_seed)\n", "system.integrator.set_vv()\n", "\n", - "p3m_params = {\"mesh\": 10}\n", + "p3m_params = {\"mesh\": 20}\n", "p3m = espressomd.electrostatics.P3M(prefactor=L_BJERRUM * KT, accuracy=2.5e-3, **p3m_params)\n", "system.electrostatics.solver = p3m\n", "system.integrator.run(steps_per_loop)\n", @@ -947,7 +955,8 @@ "source": [ "[1] Daan Frenkel, Berend Smit. Understanding Molecular Simulation: From Algorithms to Applications. 2nd edition, chapter 5: Monte Carlo Simulations in Various Ensembles, section 5.6: Grand-Canonical Ensemble, pp. 126–135. Academic Press, 2002, ISBN: 978-0-12-267351-1, doi:[10.1016/B978-012267351-1/50007-9](https://doi.org/10.1016/B978-012267351-1/50007-9). \n", "[2] N. Metropolis, A. Rosenbluth, M. Rosenbluth, A. Teller. Equation of state calculations by fast computing machines. *J. Chem. Phys.* 21, pp. 1087–1092 (1953), doi:[10.1063/1.1699114](https://doi.org/10.1063/1.1699114). \n", - "[3] F. Donnan. The theory of membrane equilibria. *Chemical Reviews* 1(1), pp. 73–90 (1924), doi:[10.1021/cr60001a003](https://doi.org/10.1021/cr60001a003). " + "[3] F. Donnan. The theory of membrane equilibria. *Chemical Reviews* 1(1), pp. 73–90 (1924), doi:[10.1021/cr60001a003](https://doi.org/10.1021/cr60001a003). \n", + "[4] J. Landsgesell, P. Hebbeker, O. Rud, R. Lunkad, P. Košovan, C. Holm. Grand-Reaction method for simulations of ionization equilibria coupled to ion partitioning. *Macromolecules* 53(8), pp. 3007–3020 (2020), doi:[10.1021/acs.macromol.0c00260](https://doi.org/10.1021/acs.macromol.0c00260). " ] } ], diff --git a/doc/tutorials/mlip-water/01_TIP4P_water.ipynb b/doc/tutorials/mlip-water/01_TIP4P_water.ipynb index 0f48f4394a..3d3ae2ff49 100644 --- a/doc/tutorials/mlip-water/01_TIP4P_water.ipynb +++ b/doc/tutorials/mlip-water/01_TIP4P_water.ipynb @@ -535,7 +535,7 @@ "[1] Schäfer, Segreto, Zills, Holm, Kästner, \"Apax: A Flexible and Performant Framework for the Development of Machine-Learned Interatomic Potentials\", *Journal of Chemical Information and Modeling*, 65(15):8066-8078, 2025, doi:[10.1021/acs.jcim.5c01221](https://doi.org/10.1021/acs.jcim.5c01221) \n", "[2] Schäfer, Segreto, Zills, Peters, Haldar, \"apax-hub/apax: v0.12.3\", Zenodo, 2025, doi:[10.5281/zenodo.17092886](https://doi.org/10.5281/zenodo.17092886) \n", "[3] Abascal, Vega \"A general purpose model for the condensed phases of water: TIP4P/2005\", *Journal of Chemical Physics*, 123(23):234505, 2005, doi:[10.1063/1.2121687](https://doi.org/10.1063/1.2121687) \n", - "[4] Soper, \"The radial distribution functions of water and ice from 220 to 673 K and at pressures up to 400 MPa\", *Chemical Physics*, 258(2–3):121–13, 2000, doi:[10.1016/S0301-0104(00)00179-8](https://doi.org/10.1016/S0301-0104(00)00179-8) \n", + "[4] Soper, \"The radial distribution functions of water and ice from 220 to 673 K and at pressures up to 400 MPa\", *Chemical Physics*, 258(2–3):121–13, 2000, doi:[10.1016/S0301-0104(00)00179-8](https://doi.org/10.1016/S0301-0104%2800%2900179-8) \n", "[5] Soper, \"The radial distribution functions of water as derived from radiation total scattering experiments: is there anything we can say for sure?\", *International Scholarly Research Notices*, 2013:279463, 2013, doi:[10.1155/2013/279463](https://doi.org/10.1155/2013/279463) " ] } diff --git a/testsuite/tutorials/test_grand_canonical_monte_carlo.py b/testsuite/tutorials/test_grand_canonical_monte_carlo.py index f7eb0a53ae..a80ecf75ca 100644 --- a/testsuite/tutorials/test_grand_canonical_monte_carlo.py +++ b/testsuite/tutorials/test_grand_canonical_monte_carlo.py @@ -23,7 +23,7 @@ tutorial, skipIfMissingFeatures = importlib_wrapper.configure_and_import( "@TUTORIALS_DIR@/grand_canonical_monte_carlo/grand_canonical_monte_carlo.py", - p3m_params={"mesh": 10, "cao": 6, "r_cut": 8.22}) + p3m_params={"mesh": 20, "cao": 4, "r_cut": 5.05}) @skipIfMissingFeatures From ed49a0e7a082827cb5edfe54d0edf2d14681163c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Grad?= Date: Mon, 13 Apr 2026 16:46:28 +0200 Subject: [PATCH 08/11] CUDA+OpenMP --- CMakeLists.txt | 6 +++++- src/core/CMakeLists.txt | 5 +++-- src/core/p3m/CMakeLists.txt | 17 +++++++++++++---- src/walberla_bridge/CMakeLists.txt | 2 ++ src/walberla_bridge/src/BoundaryPackInfo.hpp | 2 -- src/walberla_bridge/tests/CMakeLists.txt | 1 + 6 files changed, 24 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 57c1b0f1c2..2fbf7216ec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -654,7 +654,11 @@ if(ESPRESSO_BUILD_WITH_FFTW) endif() endif() -find_package(OpenMP 4.5 REQUIRED COMPONENTS CXX) +set(ESPRESSO_OPENMP_COMPONENTS CXX) +if(ESPRESSO_BUILD_WITH_CUDA) + list(APPEND ESPRESSO_OPENMP_COMPONENTS CUDA) +endif() +find_package(OpenMP 4.5 REQUIRED COMPONENTS ${ESPRESSO_OPENMP_COMPONENTS}) if(NOT EXISTS ${FETCHCONTENT_BASE_DIR}/kokkos-src) find_package(Kokkos 4.6 QUIET) diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 11e024c159..b0c6d58931 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -77,8 +77,9 @@ install(TARGETS espresso_core target_link_libraries( espresso_core PRIVATE espresso::config espresso::utils::mpi espresso::shapes - espresso::compiler_flags OpenMP::OpenMP_CXX Kokkos::kokkos - Cabana::Core + espresso::compiler_flags OpenMP::OpenMP_CXX + $<$:OpenMP::OpenMP_CUDA> + Kokkos::kokkos Cabana::Core $<$:espresso::walberla> $<$:nlopt> $<$:GSL::gsl> diff --git a/src/core/p3m/CMakeLists.txt b/src/core/p3m/CMakeLists.txt index 4d121287dd..6954c018ca 100644 --- a/src/core/p3m/CMakeLists.txt +++ b/src/core/p3m/CMakeLists.txt @@ -33,10 +33,19 @@ if(ESPRESSO_BUILD_WITH_FFTW) ) target_link_libraries( espresso_p3m - PRIVATE espresso::compiler_flags espresso::p3m::compiler_flags - espresso::instrumentation espresso::utils espresso::config - Random123 Boost::mpi MPI::MPI_CXX Heffte::Heffte OpenMP::OpenMP_CXX - Kokkos::kokkos Cabana::Core) + PRIVATE espresso::compiler_flags + espresso::p3m::compiler_flags + espresso::instrumentation + espresso::utils + espresso::config + Random123 + Boost::mpi + MPI::MPI_CXX + Heffte::Heffte + OpenMP::OpenMP_CXX + $<$:OpenMP::OpenMP_CUDA> + Kokkos::kokkos + Cabana::Core) set_target_properties(espresso_p3m PROPERTIES POSITION_INDEPENDENT_CODE ON) target_link_libraries(espresso_core PRIVATE espresso_p3m) target_sources( diff --git a/src/walberla_bridge/CMakeLists.txt b/src/walberla_bridge/CMakeLists.txt index 76947b5916..5a9a6c2dd8 100644 --- a/src/walberla_bridge/CMakeLists.txt +++ b/src/walberla_bridge/CMakeLists.txt @@ -131,12 +131,14 @@ target_link_libraries( espresso::walberla_codegen $<$:Heffte::Heffte> $<$:espresso::avx_flags> + $<$:OpenMP::OpenMP_CUDA> OpenMP::OpenMP_CXX) target_link_libraries( espresso_walberla_codegen PRIVATE espresso::compiler_flags espresso::walberla::compiler_flags espresso::walberla_codegen::compiler_flags $<$:espresso::avx_flags> + $<$:OpenMP::OpenMP_CUDA> OpenMP::OpenMP_CXX) function(espresso_set_walberla_codegen_kernel_source_properties source_file) diff --git a/src/walberla_bridge/src/BoundaryPackInfo.hpp b/src/walberla_bridge/src/BoundaryPackInfo.hpp index 3a25290417..244ccba5b4 100644 --- a/src/walberla_bridge/src/BoundaryPackInfo.hpp +++ b/src/walberla_bridge/src/BoundaryPackInfo.hpp @@ -76,9 +76,7 @@ class BoundaryPackInfo : public PackInfo { void communicateLocal(IBlock const *sender, IBlock *receiver, stencil::Direction dir) override { -#ifdef _OPENMP #pragma omp critical -#endif { mpi::SendBuffer sBuffer; packDataImpl(sender, dir, sBuffer); diff --git a/src/walberla_bridge/tests/CMakeLists.txt b/src/walberla_bridge/tests/CMakeLists.txt index cd0be82d27..ba661d90be 100644 --- a/src/walberla_bridge/tests/CMakeLists.txt +++ b/src/walberla_bridge/tests/CMakeLists.txt @@ -35,6 +35,7 @@ function(ESPRESSO_ADD_TEST) DEPENDS ${TEST_DEPENDS} $<$:espresso::avx_flags> + $<$:OpenMP::OpenMP_CUDA> OpenMP::OpenMP_CXX espresso::walberla espresso::walberla_deps From ebbdfc51aa7c8b5313421e94d13769bbaf1921b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Grad?= Date: Tue, 14 Apr 2026 12:12:46 +0200 Subject: [PATCH 09/11] Build with CUDA + OpenMP --- .github/workflows/push_pull.yml | 6 +++--- .gitlab-ci.yml | 4 ++-- CMakeLists.txt | 19 +++++++++++++------ src/walberla_bridge/src/BoundaryPackInfo.hpp | 2 -- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/.github/workflows/push_pull.yml b/.github/workflows/push_pull.yml index 96d40f6468..624da63d35 100644 --- a/.github/workflows/push_pull.yml +++ b/.github/workflows/push_pull.yml @@ -52,7 +52,7 @@ jobs: debian: runs-on: ubuntu-latest container: - image: ghcr.io/espressomd/docker/debian:a55be868226f9b1b313deea773ce01705707a6f1-base-layer + image: ghcr.io/espressomd/docker/debian:ab5bc64f5c3cc7330952f8b3bf321270ca53960c-base-layer credentials: username: ${{ github.actor }} password: ${{ secrets.github_token }} @@ -93,7 +93,7 @@ jobs: runs-on: ubuntu-latest if: ${{ github.repository == 'espressomd/espresso' }} container: - image: ghcr.io/espressomd/docker/ubuntu-wo-dependencies:a55be868226f9b1b313deea773ce01705707a6f1-base-layer + image: ghcr.io/espressomd/docker/ubuntu-wo-dependencies:ab5bc64f5c3cc7330952f8b3bf321270ca53960c-base-layer credentials: username: ${{ github.actor }} password: ${{ secrets.github_token }} @@ -135,7 +135,7 @@ jobs: runs-on: ubuntu-latest if: ${{ github.repository == 'espressomd/espresso' }} container: - image: ghcr.io/espressomd/docker/fedora:a55be868226f9b1b313deea773ce01705707a6f1-base-layer + image: ghcr.io/espressomd/docker/fedora:ab5bc64f5c3cc7330952f8b3bf321270ca53960c-base-layer credentials: username: ${{ github.actor }} password: ${{ secrets.github_token }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4182a4af78..8959167738 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: ghcr.io/espressomd/docker/ubuntu:a55be868226f9b1b313deea773ce01705707a6f1 +image: ghcr.io/espressomd/docker/ubuntu:ab5bc64f5c3cc7330952f8b3bf321270ca53960c stages: - prepare @@ -16,7 +16,7 @@ stages: .notification_job_template: ¬ification_job_definition <<: *global_job_definition - image: ghcr.io/espressomd/docker/alpine:a55be868226f9b1b313deea773ce01705707a6f1 + image: ghcr.io/espressomd/docker/alpine:ab5bc64f5c3cc7330952f8b3bf321270ca53960c variables: GET_SOURCES_ATTEMPTS: 3 dependencies: [] diff --git a/CMakeLists.txt b/CMakeLists.txt index 2fbf7216ec..e9fdcaa11c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,9 @@ # cmake_minimum_required(VERSION 3.27.6) +if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM AND ESPRESSO_BUILD_WITH_CUDA) + cmake_minimum_required(VERSION 3.31.0) +endif() cmake_policy(VERSION 3.27.6) if(POLICY CMP0167) # use BoostConfig.cmake shipped with Boost 1.70+ instead of the one in CMake @@ -620,6 +623,16 @@ endif() # Libraries # +set(ESPRESSO_OPENMP_COMPONENTS CXX) +if(ESPRESSO_BUILD_WITH_CUDA) + list(APPEND ESPRESSO_OPENMP_COMPONENTS CUDA) +endif() +# ESPResSo and Kokkos require some of the new features from OpenMP 5.0, but a +# version check cannot be enforced here, since GCC versions 12 to 15 report +# 4.5 support for the CXX component, and NVHPC versions <= 26.3 report 4.5 +# support for the CUDA component +find_package(OpenMP REQUIRED COMPONENTS ${ESPRESSO_OPENMP_COMPONENTS}) + if(ESPRESSO_BUILD_WITH_FFTW) find_package(fftw3 REQUIRED COMPONENTS omp) @@ -654,12 +667,6 @@ if(ESPRESSO_BUILD_WITH_FFTW) endif() endif() -set(ESPRESSO_OPENMP_COMPONENTS CXX) -if(ESPRESSO_BUILD_WITH_CUDA) - list(APPEND ESPRESSO_OPENMP_COMPONENTS CUDA) -endif() -find_package(OpenMP 4.5 REQUIRED COMPONENTS ${ESPRESSO_OPENMP_COMPONENTS}) - if(NOT EXISTS ${FETCHCONTENT_BASE_DIR}/kokkos-src) find_package(Kokkos 4.6 QUIET) endif() diff --git a/src/walberla_bridge/src/BoundaryPackInfo.hpp b/src/walberla_bridge/src/BoundaryPackInfo.hpp index 244ccba5b4..57f48f2b3c 100644 --- a/src/walberla_bridge/src/BoundaryPackInfo.hpp +++ b/src/walberla_bridge/src/BoundaryPackInfo.hpp @@ -27,8 +27,6 @@ #include #include -#include - #include #include #include From e8d66b02e3542f772f308bae2a02c3345518d643 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Grad?= Date: Tue, 14 Apr 2026 14:14:49 +0200 Subject: [PATCH 10/11] style --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e9fdcaa11c..9396182516 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -628,9 +628,9 @@ if(ESPRESSO_BUILD_WITH_CUDA) list(APPEND ESPRESSO_OPENMP_COMPONENTS CUDA) endif() # ESPResSo and Kokkos require some of the new features from OpenMP 5.0, but a -# version check cannot be enforced here, since GCC versions 12 to 15 report -# 4.5 support for the CXX component, and NVHPC versions <= 26.3 report 4.5 -# support for the CUDA component +# version check cannot be enforced here, since GCC versions 12 to 15 report 4.5 +# support for the CXX component, and NVHPC versions <= 26.3 report 4.5 support +# for the CUDA component find_package(OpenMP REQUIRED COMPONENTS ${ESPRESSO_OPENMP_COMPONENTS}) if(ESPRESSO_BUILD_WITH_FFTW) From dd38581ef548e5f8ce33643d076333bf1aafe44e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Grad?= Date: Tue, 14 Apr 2026 16:16:35 +0200 Subject: [PATCH 11/11] Remove unused code --- src/core/forces_inline.hpp | 290 ------------------ src/core/integrate.cpp | 6 - .../EspressoSystemStandAlone_test.cpp | 5 - testsuite/python/interactions_non-bonded.py | 36 ++- 4 files changed, 23 insertions(+), 314 deletions(-) diff --git a/src/core/forces_inline.hpp b/src/core/forces_inline.hpp index a7b10a3a07..382b28446d 100644 --- a/src/core/forces_inline.hpp +++ b/src/core/forces_inline.hpp @@ -166,162 +166,6 @@ inline ParticleForce calc_opposing_force(ParticleForce const &pf, return out; } -/** - * @brief For interactions which need particle information. - */ -inline void add_non_bonded_pair_force_with_p( - Particle &p1, Particle &p2, ParticleForce &pf, ParticleForce &p1f_asym, - ParticleForce &p2f_asym, Utils::Vector3d const &d, double dist, - double dist2, double q1q2, IA_parameters const &ia_params, - [[maybe_unused]] bool do_nonbonded_flag, - Thermostat::Thermostat const &thermostat, BoxGeometry const &box_geo, - [[maybe_unused]] BondedInteractionsMap const &bonded_ias, - [[maybe_unused]] Utils::Vector3d *const virial, - Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel, - Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel, - Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel, - Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel) { - - /***********************************************/ - /* non-bonded pair potentials */ - /***********************************************/ - - if (dist < ia_params.max_cut) { -#ifdef ESPRESSO_EXCLUSIONS - if (do_nonbonded_flag) { -#endif -#ifdef ESPRESSO_THOLE - pf.f += thole_pair_force(p1, p2, ia_params, d, dist, bonded_ias, - coulomb_kernel); -#endif - pf += calc_non_central_force(p1, p2, ia_params, d, dist); -#ifdef ESPRESSO_EXCLUSIONS - } -#endif - } - - /*********************************************************************/ - /* everything before this contributes to the virial pressure in NpT, */ - /* but nothing afterwards, since the contribution to pressure from */ - /* electrostatic is calculated by energy */ - /*********************************************************************/ -#ifdef ESPRESSO_NPT - if (virial) { - *virial += hadamard_product(pf.f, d); - } -#endif // ESPRESSO_NPT - - /***********************************************/ - /* short-range electrostatics */ - /***********************************************/ - -#ifdef ESPRESSO_ELECTROSTATICS - // real-space electrostatic charge-charge interaction - if (q1q2 != 0. and coulomb_kernel != nullptr) { - pf.f += (*coulomb_kernel)(q1q2, d, dist); -#ifdef ESPRESSO_NPT - if (virial) { - (*virial)[0] += (*coulomb_u_kernel)(p1.pos(), p2.pos(), q1q2, d, dist); - } -#endif // ESPRESSO_NPT - if (elc_kernel) { - (*elc_kernel)(p1.pos(), p2.pos(), p1f_asym.f, p2f_asym.f, q1q2); - } - } -#endif // ESPRESSO_ELECTROSTATICS - - /***********************************************/ - /* thermostat */ - /***********************************************/ - - /* The inter dpd force should not be part of the virial */ -#ifdef ESPRESSO_DPD - if (thermostat.thermo_switch & THERMO_DPD) { - auto const force = - dpd_pair_force(p1.pos(), p1.v(), p1.id(), p2.pos(), p2.v(), p2.id(), - *thermostat.dpd, box_geo, ia_params, d, dist, dist2); - pf += force; - } -#endif - - /***********************************************/ - /* short-range magnetostatics */ - /***********************************************/ - -#ifdef ESPRESSO_DIPOLES - // real-space magnetic dipole-dipole - if (dipoles_kernel) { - auto const d1d2 = p1.dipm() * p2.dipm(); - if (d1d2 != 0.) { - pf += - (*dipoles_kernel)(d1d2, p1.calc_dip(), p2.calc_dip(), d, dist, dist2); - } - } -#endif -} - -/** Calculate non-bonded forces between a pair of particles and update their - * forces and torques. - * @param[in,out] p1 particle 1. - * @param[in,out] p2 particle 2. - * @param[in] d vector between @p p1 and @p p2. - * @param[in] dist distance between @p p1 and @p p2. - * @param[in] dist2 distance squared between @p p1 and @p p2. - * @param[in] q1q2 charge x charge between @p p1 and @p p2. - * @param[in] ia_params non-bonded interaction kernels. - * @param[in] thermostat thermostat. - * @param[in] box_geo box geometry. - * @param[in] bonded_ias bonded interaction kernels. - * @param[out] virial NpT virial. - * @param[in] coulomb_kernel Coulomb force kernel. - * @param[in] dipoles_kernel Dipolar force kernel. - * @param[in] elc_kernel ELC force correction kernel. - * @param[in] coulomb_u_kernel Coulomb energy kernel. - */ -inline void add_non_bonded_pair_force( - Particle &p1, Particle &p2, Utils::Vector3d const &d, double dist, - double dist2, double q1q2, IA_parameters const &ia_params, - Thermostat::Thermostat const &thermostat, BoxGeometry const &box_geo, - [[maybe_unused]] BondedInteractionsMap const &bonded_ias, - [[maybe_unused]] Utils::Vector3d *const virial, - Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel, - Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel, - Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel, - Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel) { - - ParticleForce pf{}; - ParticleForce p1f_asym{}; - ParticleForce p2f_asym{}; - -#ifdef ESPRESSO_EXCLUSIONS - auto const do_nonbonded_flag = do_nonbonded(p1, p2); -#else - auto constexpr do_nonbonded_flag = true; -#endif - - if (dist < ia_params.max_cut) { -#ifdef ESPRESSO_EXCLUSIONS - if (do_nonbonded_flag) { -#endif - pf.f += calc_central_radial_force(ia_params, d, dist); -#ifdef ESPRESSO_EXCLUSIONS - } -#endif - } - - add_non_bonded_pair_force_with_p( - p1, p2, pf, p1f_asym, p2f_asym, d, dist, dist2, q1q2, ia_params, - do_nonbonded_flag, thermostat, box_geo, bonded_ias, virial, - coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel); - - /***********************************************/ - /* add total non-bonded forces to particles */ - /***********************************************/ - - p1.force_and_torque() += pf + p1f_asym; - p2.force_and_torque() += calc_opposing_force(pf, d) + p2f_asym; -} - /** Compute the bonded interaction force between particle pairs. * * @param[in] iaparams Bonded parameters for the interaction. @@ -367,45 +211,6 @@ inline std::optional calc_bond_pair_force( throw BondUnknownTypeError(); } -inline bool add_bonded_two_body_force( - Bonded_IA_Parameters const &iaparams, BoxGeometry const &box_geo, - Particle &p1, Particle &p2, [[maybe_unused]] Utils::Vector3d *const virial, - Coulomb::ShortRangeForceKernel::kernel_type const *kernel) { - auto const dx = box_geo.get_mi_vector(p1.pos(), p2.pos()); - - if (auto const *iap = std::get_if(&iaparams)) { - auto result = iap->forces(p1, p2, dx); - if (result) { - auto const &forces = result.value(); - - p1.force() += std::get<0>(forces); - p2.force() += std::get<1>(forces); - - return false; - } - } else { - auto result = calc_bond_pair_force(iaparams, dx, -#ifdef ESPRESSO_ELECTROSTATICS - p1.q() * p2.q(), kernel -#else - 0.0, nullptr -#endif - ); - if (result) { - p1.force() += result.value(); - p2.force() -= result.value(); - -#ifdef ESPRESSO_NPT - if (virial) { - *virial += hadamard_product(result.value(), dx); - } -#endif - return false; - } - } - return true; -} - ESPRESSO_ATTR_ALWAYS_INLINE inline std::optional< std::tuple> @@ -432,28 +237,6 @@ calc_bonded_three_body_force(Bonded_IA_Parameters const &iaparams, throw BondUnknownTypeError(); } -inline bool add_bonded_three_body_force(Bonded_IA_Parameters const &iaparams, - BoxGeometry const &box_geo, - Particle &p1, Particle &p2, - Particle &p3) { - if (std::get_if(&iaparams)) { - return false; - } - auto const vec1 = box_geo.get_mi_vector(p2.pos(), p1.pos()); - auto const vec2 = box_geo.get_mi_vector(p3.pos(), p1.pos()); - auto const result = calc_bonded_three_body_force(iaparams, vec1, vec2); - if (result) { - auto const &forces = result.value(); - - p1.force() += std::get<0>(forces); - p2.force() += std::get<1>(forces); - p3.force() += std::get<2>(forces); - - return false; - } - return true; -} - ESPRESSO_ATTR_ALWAYS_INLINE inline std::optional> @@ -488,76 +271,3 @@ calc_bonded_four_body_force( #endif throw BondUnknownTypeError(); } - -inline bool add_bonded_four_body_force(Bonded_IA_Parameters const &iaparams, - BoxGeometry const &box_geo, Particle &p1, - Particle &p2, Particle &p3, - Particle &p4) { - auto const pos1 = p1.pos(); - auto const pos2 = p2.pos(); - auto const pos3 = p3.pos(); - auto const pos4 = p4.pos(); - auto const vel1 = p1.v(); - auto const vel3 = p3.v(); - auto const image1 = p1.image_box(); - auto const result = calc_bonded_four_body_force( - iaparams, box_geo, pos1, pos2, pos3, pos4, vel1, vel3, image1); - if (result) { - auto const &forces = result.value(); - - p1.force() += std::get<0>(forces); - p2.force() += std::get<1>(forces); - p3.force() += std::get<2>(forces); - p4.force() += std::get<3>(forces); - - return false; - } - - return true; -} - -inline bool -add_bonded_force(Particle &p1, int bond_id, std::span partners, - BondedInteractionsMap const &bonded_ia_params, - BondBreakage::BondBreakage &bond_breakage, - BoxGeometry const &box_geo, - [[maybe_unused]] Utils::Vector3d *const virial, - Coulomb::ShortRangeForceKernel::kernel_type const *kernel) { - - auto const n_partners = static_cast(partners.size()); - - // Consider for bond breakage - if (n_partners == 1) { // pair bonds - auto d = box_geo.get_mi_vector(p1.pos(), partners[0]->pos()).norm(); - if (bond_breakage.check_and_handle_breakage( - p1.id(), {{partners[0]->id(), std::nullopt}}, bond_id, d)) { - return false; - } - } - if (n_partners == 2) { // angle bond - auto d = - box_geo.get_mi_vector(partners[0]->pos(), partners[1]->pos()).norm(); - if (bond_breakage.check_and_handle_breakage( - p1.id(), {{partners[0]->id(), partners[1]->id()}}, bond_id, d)) { - return false; - } - } - - auto const &iaparams = *bonded_ia_params.at(bond_id); - - switch (n_partners) { - case 0: - return false; - case 1: - return add_bonded_two_body_force(iaparams, box_geo, p1, *partners[0], - virial, kernel); - case 2: - return add_bonded_three_body_force(iaparams, box_geo, p1, *partners[0], - *partners[1]); - case 3: - return add_bonded_four_body_force(iaparams, box_geo, p1, *partners[0], - *partners[1], *partners[2]); - default: - throw BondInvalidSizeError{n_partners}; - } -} diff --git a/src/core/integrate.cpp b/src/core/integrate.cpp index 4a3c134efd..c796a1b766 100644 --- a/src/core/integrate.cpp +++ b/src/core/integrate.cpp @@ -755,9 +755,6 @@ int System::System::integrate(int n_steps, int reuse_forces) { #ifdef ESPRESSO_CALIPER CALI_MARK_END("lb_propagation"); #endif -#ifdef ESPRESSO_FPE - auto const trap_pause = fe_trap::make_shared_pause_scoped(); -#endif #ifdef ESPRESSO_CALIPER CALI_MARK_BEGIN("ek_propagation"); #endif @@ -784,9 +781,6 @@ int System::System::integrate(int n_steps, int reuse_forces) { propagation.ek_skipped_md_steps += 1; if (propagation.ek_skipped_md_steps >= md_steps_per_ek_step) { propagation.ek_skipped_md_steps = 0; -#ifdef ESPRESSO_FPE - auto const trap_pause = fe_trap::make_shared_pause_scoped(); -#endif #ifdef ESPRESSO_CALIPER CALI_MARK_BEGIN("ek_propagation"); #endif diff --git a/src/core/unit_tests/EspressoSystemStandAlone_test.cpp b/src/core/unit_tests/EspressoSystemStandAlone_test.cpp index a1d467c138..4a6b3f082c 100644 --- a/src/core/unit_tests/EspressoSystemStandAlone_test.cpp +++ b/src/core/unit_tests/EspressoSystemStandAlone_test.cpp @@ -710,11 +710,6 @@ BOOST_FIXTURE_TEST_CASE(espresso_system_stand_alone, ParticleFactory) { BOOST_CHECK_THROW(force_kernel(1u), BondUnknownTypeError); BOOST_CHECK_THROW(force_kernel(2u), BondUnknownTypeError); BOOST_CHECK_THROW(force_kernel(3u), BondUnknownTypeError); - BOOST_CHECK_THROW(add_bonded_force(plist[0], 0, {plist_ptr.begin(), 5ul}, - *system.bonded_ias, - *system.bond_breakage, *system.box_geo, - nullptr, nullptr), - BondInvalidSizeError); #ifdef ESPRESSO_CUDA BOOST_CHECK_THROW( invoke_skip_cuda_exceptions([]() { throw std::runtime_error(""); }), diff --git a/testsuite/python/interactions_non-bonded.py b/testsuite/python/interactions_non-bonded.py index 93ad53ade7..f16886db9d 100644 --- a/testsuite/python/interactions_non-bonded.py +++ b/testsuite/python/interactions_non-bonded.py @@ -630,28 +630,38 @@ def get_reference_torque(gb_params, r, dir1, dir2): # force equals minus the counter-force np.testing.assert_array_equal(f1_sim, -f2_sim) # compare force to reference force - for i in range(3): - self.assertAlmostEqual(f2_sim[i], f2_ref[i], delta=delta) + np.testing.assert_allclose(f2_sim, f2_ref, atol=delta, rtol=0.) + # Test pressure tensor + p_ref = np.einsum("i,j", r, f2_ref) / self.system.volume() + p_sim = self.system.analysis.pressure_tensor()["non_bonded"] + np.testing.assert_allclose(p_sim, p_ref, atol=delta, rtol=0.) # Calc torques - torque1_sim = p1.torque_lab - torque2_sim = p2.torque_lab + torque1_sim = np.copy(p1.torque_lab) + torque2_sim = np.copy(p2.torque_lab) torque1_ref = get_reference_torque( gb_params, r, director1, director2) torque2_ref = get_reference_torque( gb_params, r, director2, director1) # Test torques - for i in range(3): - self.assertAlmostEqual( - torque1_sim[i], - torque1_ref[i], - delta=delta) - self.assertAlmostEqual( - torque2_sim[i], - torque2_ref[i], - delta=delta) + np.testing.assert_allclose( + torque1_sim, torque1_ref, atol=delta, rtol=0.) + np.testing.assert_allclose( + torque2_sim, torque2_ref, atol=delta, rtol=0.) + + # Test beyond cutoff + old_pos2 = np.copy(p2.pos) + p2.pos = p1.pos + [1.01 * cut, 0., 0.] + self.system.integrator.run(recalc_forces=True, steps=0) + np.testing.assert_array_equal(np.copy(p1.f), [0., 0., 0.]) + np.testing.assert_array_equal(np.copy(p2.f), [0., 0., 0.]) + np.testing.assert_array_equal(np.copy(p1.torque_lab), [0., 0., 0.]) + np.testing.assert_array_equal(np.copy(p2.torque_lab), [0., 0., 0.]) + np.testing.assert_array_equal( + self.system.analysis.pressure_tensor()["non_bonded"], 0.) # Test zero energy + p2.pos = old_pos2 self.system.non_bonded_inter[0, 0].gay_berne.set_params( sig=sigma_0, cut=0, eps=0, k1=k_1, k2=k_2, mu=mu, nu=nu) self.system.integrator.run(0)