diff --git a/var/spack/repos/builtin/packages/hpx-kokkos/package.py b/var/spack/repos/builtin/packages/hpx-kokkos/package.py new file mode 100644 index 0000000000..5c3ed3aeea --- /dev/null +++ b/var/spack/repos/builtin/packages/hpx-kokkos/package.py @@ -0,0 +1,49 @@ +# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + + +class HpxKokkos(CMakePackage, CudaPackage, ROCmPackage): + """HPXKokkos is an interoperability library for HPX and Kokkos""" + + homepage = 'https://github.com/STEllAR-GROUP/hpx-kokkos' + url = "https://github.com/STEllAR-GROUP/hpx-kokkos/archive/0.0.0.tar.gz" + maintainers = ['G-071', 'msimberg'] + + version('master', git='https://github.com/STEllAR-GROUP/hpx-kokkos.git', branch='master') + version('0.3.0', sha256='83c1d11dab95552ad0abdae767c71f757811d7b51d82bd231653dc942e89a45d') + version('0.2.0', sha256='289b711cea26afe80be002fc521234c9194cd0e8f69863f3b08b654674dbe5d5') + version('0.1.0', sha256='24edb817d0969f4aea1b68eab4984c2ea9a58f4760a9b8395e20f85b178f0850') + + cxxstds = ('14', '17', '20') + variant('cxxstd', default='14', values=cxxstds, description='Use the specified C++ standard when building.') + + depends_on('cmake@3.19:', type='build') + + depends_on('hpx') + depends_on('kokkos +hpx +hpx_async_dispatch') + + depends_on('hpx@1.8:', when='@0.3:') + depends_on('kokkos@3.6:', when='@0.3:') + + depends_on('hpx@1.7', when='@0.2') + depends_on('kokkos@3.6:', when='@0.2') + + depends_on('hpx@1.6', when='@0.1') + depends_on('kokkos@3.2:3.5', when='@0.1') + + for cxxstd in cxxstds: + depends_on('hpx cxxstd={0}'.format(cxxstd), when='cxxstd={0}'.format(cxxstd)) + depends_on('kokkos std={0}'.format(cxxstd), when='cxxstd={0}'.format(cxxstd)) + + # HPXKokkos explicitly supports CUDA and ROCm. Other GPU backends can be + # used but without support in HPXKokkos. Other CPU backends, except Serial, + # can't be used together with the HPX backend. + depends_on('hpx +cuda', when='+cuda') + depends_on('kokkos +cuda +cuda_lambda +cuda_constexpr', when='+cuda') + + depends_on('hpx +rocm', when='+rocm') + depends_on('kokkos +rocm', when='+rocm') diff --git a/var/spack/repos/builtin/packages/kokkos/hpx_profiling_fences.patch b/var/spack/repos/builtin/packages/kokkos/hpx_profiling_fences.patch new file mode 100644 index 0000000000..013fbd9f67 --- /dev/null +++ b/var/spack/repos/builtin/packages/kokkos/hpx_profiling_fences.patch @@ -0,0 +1,84 @@ +diff --git a/core/src/HPX/Kokkos_HPX_Task.hpp b/core/src/HPX/Kokkos_HPX_Task.hpp +index 7bb3ca5d0..ff50fdc5f 100644 +--- a/core/src/HPX/Kokkos_HPX_Task.hpp ++++ b/core/src/HPX/Kokkos_HPX_Task.hpp +@@ -216,7 +216,7 @@ class TaskQueueSpecializationConstrained< + task_queue.scheduler = &scheduler; + Kokkos::Impl::dispatch_execute_task(&task_queue, + Kokkos::Experimental::HPX()); +- Kokkos::Experimental::HPX().fence()"Kokkos::Impl::TaskQueueSpecializationConstrained::execute: fence after task execution"; ++ Kokkos::Experimental::HPX().fence("Kokkos::Impl::TaskQueueSpecializationConstrained::execute: fence after task execution"); + } + + // Must provide task queue execution function +diff --git a/core/src/Kokkos_HPX.hpp b/core/src/Kokkos_HPX.hpp +index 236211864..3e8522e94 100644 +--- a/core/src/Kokkos_HPX.hpp ++++ b/core/src/Kokkos_HPX.hpp +@@ -282,11 +282,11 @@ class HPX { + m_mode = other.m_mode; + m_independent_instance_data = other.m_independent_instance_data; + m_buffer = m_mode == instance_mode::independent +- ? m_independent_instance_data->m_buffer +- : m_global_instance_data.m_buffer; +- m_future = m_mode == instance_mode::independent +- ? m_independent_instance_data->m_future +- : m_global_instance_data.m_future; ++ ? m_independent_instance_data->m_buffer ++ : m_global_instance_data.m_buffer; ++ m_future = m_mode == instance_mode::independent ++ ? m_independent_instance_data->m_future ++ : m_global_instance_data.m_future; + return *this; + } + #else +@@ -322,25 +322,36 @@ class HPX { + "Fence"); + } + void impl_fence_instance(const std::string &name) const { +- Kokkos::Tools::Experimental::Impl::profile_fence_event(name, *this, [&]() { +- if (hpx::threads::get_self_ptr() == nullptr) { +- hpx::threads::run_as_hpx_thread([this]() { impl_get_future().wait(); }); +- } else { +- impl_get_future().wait(); +- } +- }); ++ Kokkos::Tools::Experimental::Impl::profile_fence_event< ++ Kokkos::Experimental::HPX>( ++ name, ++ Kokkos::Tools::Experimental::Impl::DirectFenceIDHandle{ ++ impl_instance_id()}, ++ [&]() { ++ if (hpx::threads::get_self_ptr() == nullptr) { ++ hpx::threads::run_as_hpx_thread( ++ [this]() { impl_get_future().wait(); }); ++ } else { ++ impl_get_future().wait(); ++ } ++ }); + } + + void impl_fence_all_instances() const { +- impl_fence_instance( ++ impl_fence_all_instances( + "Kokkos::Experimental::HPX::impl_fence_all_instances: Unnamed Global " + "HPX Fence"); + } +- void impl_fence_all_instances(const std::string &namename) const { +- Kokkos::Tools::Experimental::Impl::profile_fence_event(name, *this, [&]() { +- hpx::util::yield_while( +- []() { return m_active_parallel_region_count.load() != 0; }); +- }); ++ void impl_fence_all_instances(const std::string &name) const { ++ Kokkos::Tools::Experimental::Impl::profile_fence_event< ++ Kokkos::Experimental::HPX>( ++ name, ++ Kokkos::Tools::Experimental::SpecialSynchronizationCases:: ++ GlobalDeviceSynchronization, ++ [&]() { ++ hpx::util::yield_while( ++ []() { return m_active_parallel_region_count.load() != 0; }); ++ }); + } + #endif + diff --git a/var/spack/repos/builtin/packages/kokkos/package.py b/var/spack/repos/builtin/packages/kokkos/package.py index aea3eca8b0..fe3e582006 100644 --- a/var/spack/repos/builtin/packages/kokkos/package.py +++ b/var/spack/repos/builtin/packages/kokkos/package.py @@ -197,6 +197,13 @@ class Kokkos(CMakePackage, CudaPackage, ROCmPackage): for std in stds: depends_on('hpx cxxstd={0}'.format(std), when='+hpx std={0}'.format(std)) + # HPX version constraints + depends_on("hpx@:1.6", when="@:3.5 +hpx") + depends_on("hpx@1.7:", when="@3.6: +hpx") + + # Patches + patch("hpx_profiling_fences.patch", when="@3.5.00 +hpx") + variant('shared', default=True, description='Build shared libraries') @classmethod