Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 44 additions & 8 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,12 @@ option(FINE_TUNE "Fine-tune for the host CPU (uses -march=native; produces non-p
# its headers from its own install layout).
option(HPTT_INSTALL "Generate install rules for HPTT" ON)

# HPTT_BUILD_SHARED: ON by default so standalone builds keep producing
# libhptt.so / libhptt.dylib / hptt.dll. Parent projects that only ever
# link the static archive can set it OFF to halve build time and to skip
# the shared library entirely.
option(HPTT_BUILD_SHARED "Build the shared HPTT library (hptt_dyn)" ON)

# -----------------------------------------------------------------------------
# Build flags split by *visibility* to consumers.
#
Expand Down Expand Up @@ -69,10 +75,19 @@ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "PGI")
endif()

if(ENABLE_AVX)
list(APPEND HPTT_PRIVATE_COMPILE_OPTIONS -mavx)
if(MSVC)
list(APPEND HPTT_PRIVATE_COMPILE_OPTIONS /arch:AVX)
else()
list(APPEND HPTT_PRIVATE_COMPILE_OPTIONS -mavx)
endif()
list(APPEND HPTT_PUBLIC_COMPILE_DEFINITIONS HPTT_ARCH_AVX)
elseif(ENABLE_ARM)
list(APPEND HPTT_PRIVATE_COMPILE_OPTIONS -mfpu=neon)
# MSVC targeting ARM64 needs no opt-in flag: Advanced SIMD (Neon) is
# part of the baseline ISA there, and cl's /arch: only selects newer
# extensions.
if(NOT MSVC)
list(APPEND HPTT_PRIVATE_COMPILE_OPTIONS -mfpu=neon)
endif()
list(APPEND HPTT_PUBLIC_COMPILE_DEFINITIONS HPTT_ARCH_ARM)
elseif(ENABLE_IBM)
list(APPEND HPTT_PRIVATE_COMPILE_OPTIONS -mtune=native -maltivec -mabi=altivec)
Expand Down Expand Up @@ -123,15 +138,36 @@ endfunction()
add_library(hptt_static STATIC ${HPTT_SRCS})
hptt_configure_target(hptt_static)

add_library(hptt_dyn SHARED ${HPTT_SRCS})
hptt_configure_target(hptt_dyn)

# Namespaced ALIAS targets. add_subdirectory() consumers can now use the
# same `hptt::hptt_static` / `hptt::hptt_dyn` names that find_package(hptt)
# consumers would, so swapping integration mechanisms doesn't require
# rewriting target_link_libraries() calls.
add_library(hptt::hptt_static ALIAS hptt_static)
add_library(hptt::hptt_dyn ALIAS hptt_dyn)

# Both the export() and install(TARGETS) calls below iterate this list so
# hptt_dyn drops out of them cleanly when HPTT_BUILD_SHARED=OFF.
set(HPTT_TARGETS hptt_static)

if(HPTT_BUILD_SHARED)
add_library(hptt_dyn SHARED ${HPTT_SRCS})
hptt_configure_target(hptt_dyn)
add_library(hptt::hptt_dyn ALIAS hptt_dyn)
list(APPEND HPTT_TARGETS hptt_dyn)

# No symbol in HPTT carries __declspec(dllexport), so on Windows the DLL
# would export nothing and the linker would emit no import library at
# all -- leaving install(TARGETS ... ARCHIVE) with nothing to install.
set_target_properties(hptt_dyn PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON)

# On Windows both the static library and the DLL's import library are
# ARCHIVE artifacts named <OUTPUT_NAME>.lib, so the shared OUTPUT_NAME
# of `hptt` makes the two targets claim the same path and Ninja aborts
# with "multiple rules generate hptt.lib". Renaming just the import
# library keeps the DLL itself hptt.dll and the static archive
# hptt.lib. ARCHIVE_OUTPUT_NAME is ignored on ELF/Mach-O platforms, so
# libhptt.a / libhptt.so / libhptt.dylib are unaffected.
set_target_properties(hptt_dyn PROPERTIES ARCHIVE_OUTPUT_NAME hptt_dyn)
endif()

# Generate hpttConfig.cmake into the build dir unconditionally so the
# build-tree export below works even when HPTT_INSTALL=OFF. Parent
Expand All @@ -149,12 +185,12 @@ configure_package_config_file(
# the binary dir. Uses export(TARGETS ...) rather than export(EXPORT
# hpttTargets) so it works without install rules (i.e. when
# HPTT_INSTALL=OFF).
export(TARGETS hptt_static hptt_dyn
export(TARGETS ${HPTT_TARGETS}
FILE "${CMAKE_CURRENT_BINARY_DIR}/hpttTargets.cmake"
NAMESPACE hptt::)

if(HPTT_INSTALL)
install(TARGETS hptt_static hptt_dyn
install(TARGETS ${HPTT_TARGETS}
EXPORT hpttTargets
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
Expand Down
7 changes: 7 additions & 0 deletions include/hptt.h
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,12 @@ void dTensorTranspose( const int *perm, const int dim,
const double beta, double *B, const int *outerSizeB,
const int numThreads, const int useRowMajor = 0);

// MSVC has no C99 `_Complex` type specifier when compiling C++: in C++ mode
// <complex.h> resolves to the STL header, which forwards to <complex> and
// never declares the UCRT's _Fcomplex / _Dcomplex. There is no portable
// spelling for these two signatures, so they are declared only where
// `_Complex` is understood.
#if !defined(_MSC_VER) || defined(__clang__)
void cTensorTranspose( const int *perm, const int dim,
const float _Complex alpha, bool conjA, const float _Complex *A, const int *sizeA, const int *outerSizeA,
const float _Complex beta, float _Complex *B, const int *outerSizeB,
Expand All @@ -310,4 +316,5 @@ void zTensorTranspose( const int *perm, const int dim,
const double _Complex alpha, bool conjA, const double _Complex *A, const int *sizeA, const int *outerSizeA,
const double _Complex beta, double _Complex *B, const int *outerSizeB,
const int numThreads, const int useRowMajor = 0);
#endif
}
15 changes: 15 additions & 0 deletions include/hptt_types.h
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,21 @@
#define REGISTER_BITS 128 // ARM
#endif

// `restrict` is a C99 keyword with no C++ equivalent, so every compiler
// exposes it under its own spelling: GCC, Clang and ICC accept
// `__restrict__`, MSVC only accepts `__restrict`. clang-cl defines
// _MSC_VER but still understands the GNU spelling.
//
// This lives in hptt_types.h rather than macros.h because transpose.h --
// a public, installed header -- uses it on the A_/B_ data members.
// macros.h additionally defines the unprefixed `INLINE`, which must not
// leak into consumers' translation units.
#if defined(_MSC_VER) && !defined(__clang__)
#define HPTT_RESTRICT __restrict
#else
#define HPTT_RESTRICT __restrict__
#endif

namespace hptt {

/**
Expand Down
2 changes: 2 additions & 0 deletions include/macros.h
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
#define INLINE __forceinline
#elif defined(__GNUC__) || defined(__GNUG__)
#define INLINE __attribute__((always_inline)) inline
#elif defined(_MSC_VER)
#define INLINE __forceinline
#endif

#ifdef _OPENMP
Expand Down
4 changes: 2 additions & 2 deletions include/transpose.h
Original file line number Diff line number Diff line change
Expand Up @@ -247,8 +247,8 @@ class Transpose
void executeEstimate(const Plan *plan) noexcept; // almost identical to execute, but it just executes few iterations and then extrapolates
double getTimeLimit() const;

const floatType* __restrict__ A_; //!< rawdata pointer for A
floatType* __restrict__ B_; //!< rawdata pointer for B
const floatType* HPTT_RESTRICT A_; //!< rawdata pointer for A
floatType* HPTT_RESTRICT B_; //!< rawdata pointer for B
floatType alpha_; //!< scaling factor for A
floatType beta_; //!< scaling factor for B
int dim_; //!< dimension of the tensor
Expand Down
59 changes: 32 additions & 27 deletions src/transpose.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ namespace hptt {
template <typename floatType, int betaIsZero, bool conjA>
struct micro_kernel
{
static void execute(const floatType* __restrict__ A, const size_t lda, floatType* __restrict__ B, const size_t ldb, const floatType alpha, const floatType beta)
static void execute(const floatType* HPTT_RESTRICT A, const size_t lda, floatType* HPTT_RESTRICT B, const size_t ldb, const floatType alpha, const floatType beta)
{
constexpr int n = (REGISTER_BITS/8) / sizeof(floatType);

Expand Down Expand Up @@ -77,7 +77,7 @@ static INLINE void prefetch(const floatType* A, const int lda)
template <int betaIsZero, bool conjA>
struct micro_kernel<double, betaIsZero, conjA>
{
static void execute(const double* __restrict__ A, const size_t lda, double* __restrict__ B, const size_t ldb, const double alpha ,const double beta)
static void execute(const double* HPTT_RESTRICT A, const size_t lda, double* HPTT_RESTRICT B, const size_t ldb, const double alpha ,const double beta)
{
__m256d reg_alpha = _mm256_set1_pd(alpha); // do not alter the content of B
__m256d reg_beta = _mm256_set1_pd(beta); // do not alter the content of B
Expand Down Expand Up @@ -134,7 +134,7 @@ struct micro_kernel<double, betaIsZero, conjA>
template <int betaIsZero, bool conjA>
struct micro_kernel<float, betaIsZero, conjA>
{
static void execute(const float* __restrict__ A, const size_t lda, float* __restrict__ B, const size_t ldb, const float alpha ,const float beta)
static void execute(const float* HPTT_RESTRICT A, const size_t lda, float* HPTT_RESTRICT B, const size_t ldb, const float alpha ,const float beta)
{
__m256 reg_alpha = _mm256_set1_ps(alpha); // do not alter the content of B
__m256 reg_beta = _mm256_set1_ps(beta); // do not alter the content of B
Expand Down Expand Up @@ -246,7 +246,7 @@ static INLINE void prefetch(const floatType* A, const int lda) { }
template <int betaIsZero, bool conjA>
struct micro_kernel<float, betaIsZero, conjA>
{
static void execute(const float* __restrict__ A, const size_t lda, float* __restrict__ B, const size_t ldb, const float alpha ,const float beta)
static void execute(const float* HPTT_RESTRICT A, const size_t lda, float* HPTT_RESTRICT B, const size_t ldb, const float alpha ,const float beta)
{
float32x4_t reg_alpha = vdupq_n_f32(alpha);
float32x4_t reg_beta = vdupq_n_f32(beta);
Expand Down Expand Up @@ -304,7 +304,7 @@ struct micro_kernel<float, betaIsZero, conjA>
//template <int betaIsZero>
//struct micro_kernel<float, betaIsZero>
//{
// static void execute(const float* __restrict__ A, const size_t lda, float* __restrict__ B, const size_t ldb, const float alpha ,const float beta)
// static void execute(const float* HPTT_RESTRICT A, const size_t lda, float* HPTT_RESTRICT B, const size_t ldb, const float alpha ,const float beta)
// {
// vector float reg_alpha = vec_splats(alpha);
//
Expand Down Expand Up @@ -368,8 +368,8 @@ struct micro_kernel<float, betaIsZero, conjA>


template<int betaIsZero, typename floatType, bool conjA>
static INLINE void macro_kernel_scalar(const floatType* __restrict__ A, const size_t lda, int blockingA,
floatType* __restrict__ B, const size_t ldb, int blockingB,
static INLINE void macro_kernel_scalar(const floatType* HPTT_RESTRICT A, const size_t lda, int blockingA,
floatType* HPTT_RESTRICT B, const size_t ldb, int blockingB,
const floatType alpha ,const floatType beta)
{
#ifdef DEBUG
Expand All @@ -393,8 +393,8 @@ static INLINE void macro_kernel_scalar(const floatType* __restrict__ A, const si
}

template<int blockingA, int blockingB, int betaIsZero, typename floatType, bool useStreamingStores_, bool conjA>
static INLINE void macro_kernel(const floatType* __restrict__ A, const floatType* __restrict__ Anext, const size_t lda,
floatType* __restrict__ B, const floatType* __restrict__ Bnext, const size_t ldb,
static INLINE void macro_kernel(const floatType* HPTT_RESTRICT A, const floatType* HPTT_RESTRICT Anext, const size_t lda,
floatType* HPTT_RESTRICT B, const floatType* HPTT_RESTRICT Bnext, const size_t ldb,
const floatType alpha ,const floatType beta)
{
constexpr int blocking_micro_ = REGISTER_BITS/8 / sizeof(floatType);
Expand Down Expand Up @@ -569,8 +569,8 @@ static INLINE void macro_kernel(const floatType* __restrict__ A, const floatType
}

template<int betaIsZero, typename floatType, bool conjA>
void transpose_int_scalar( const floatType* __restrict__ A, int sizeStride1A,
floatType* __restrict__ B, int sizeStride1B, const floatType alpha, const floatType beta, const ComputeNode* plan)
void transpose_int_scalar( const floatType* HPTT_RESTRICT A, int sizeStride1A,
floatType* HPTT_RESTRICT B, int sizeStride1B, const floatType alpha, const floatType beta, const ComputeNode* plan)
{
const int32_t end = plan->end;
const size_t lda = plan->lda;
Expand Down Expand Up @@ -600,8 +600,8 @@ void transpose_int_scalar( const floatType* __restrict__ A, int sizeStride1A,
}
}
template<int blockingA, int blockingB, int betaIsZero, typename floatType, bool useStreamingStores, bool conjA>
void transpose_int( const floatType* __restrict__ A, const floatType* __restrict__ Anext,
floatType* __restrict__ B, const floatType* __restrict__ Bnext, const floatType alpha, const floatType beta,
void transpose_int( const floatType* HPTT_RESTRICT A, const floatType* HPTT_RESTRICT Anext,
floatType* HPTT_RESTRICT B, const floatType* HPTT_RESTRICT Bnext, const floatType alpha, const floatType beta,
const ComputeNode* plan)
{
const int32_t end = plan->end - (plan->inc - 1);
Expand Down Expand Up @@ -681,7 +681,7 @@ void transpose_int( const floatType* __restrict__ A, const floatType* __restrict
}

template<int betaIsZero, typename floatType, bool useStreamingStores, bool conjA>
void transpose_int_constStride1( const floatType* __restrict__ A, floatType* __restrict__ B, const floatType alpha, const floatType beta,
void transpose_int_constStride1( const floatType* HPTT_RESTRICT A, floatType* HPTT_RESTRICT B, const floatType alpha, const floatType beta,
const ComputeNode* plan)
{
const int32_t end = plan->end - (plan->inc - 1);
Expand Down Expand Up @@ -752,12 +752,15 @@ void transpose_int_constStride1( const floatType* __restrict__ A, floatType* __r
#ifdef _OPENMP
omp_init_lock(&writelock);
#endif
int tmpPerm[dim];
int tmpSizeA[dim];
int tmpOuterSizeA[dim];
int tmpOuterSizeB[dim];
accountForRowMajor(sizeA, outerSizeA, outerSizeB, perm,
tmpSizeA, tmpOuterSizeA, tmpOuterSizeB, tmpPerm, dim, useRowMajor);
// dim is a runtime value, and variable-length arrays are a GNU
// extension that MSVC rejects (C2131).
std::vector<int> tmpPerm(dim);
std::vector<int> tmpSizeA(dim);
std::vector<int> tmpOuterSizeA(dim);
std::vector<int> tmpOuterSizeB(dim);
accountForRowMajor(sizeA, outerSizeA, outerSizeB, perm,
tmpSizeA.data(), tmpOuterSizeA.data(), tmpOuterSizeB.data(),
tmpPerm.data(), dim, useRowMajor);

sizeA_.resize(dim);
perm_.resize(dim);
Expand All @@ -775,10 +778,10 @@ void transpose_int_constStride1( const floatType* __restrict__ A, floatType* __r
threadIds_.push_back(i);
}

verifyParameter(tmpSizeA, tmpPerm, tmpOuterSizeA, tmpOuterSizeB, dim);
verifyParameter(tmpSizeA.data(), tmpPerm.data(), tmpOuterSizeA.data(), tmpOuterSizeB.data(), dim);

// initializes dim_, outerSizeA, outerSizeB, sizeA and perm
skipIndices(tmpSizeA, tmpPerm, tmpOuterSizeA, tmpOuterSizeB, dim);
// initializes dim_, outerSizeA, outerSizeB, sizeA and perm
skipIndices(tmpSizeA.data(), tmpPerm.data(), tmpOuterSizeA.data(), tmpOuterSizeB.data(), dim);
fuseIndices();

// initializes lda_ and ldb_
Expand Down Expand Up @@ -867,7 +870,7 @@ void Transpose<floatType>::executeEstimate(const Plan *plan) noexcept


template<int betaIsZero, typename floatType, bool useStreamingStores, bool spawnThreads, bool conjA>
static void axpy_1D( const floatType* __restrict__ A, floatType* __restrict__ B, const int myStart, const int myEnd, const floatType alpha, const floatType beta, int numThreads)
static void axpy_1D( const floatType* HPTT_RESTRICT A, floatType* HPTT_RESTRICT B, const int myStart, const int myEnd, const floatType alpha, const floatType beta, int numThreads)
{
if( !betaIsZero )
{
Expand Down Expand Up @@ -900,8 +903,8 @@ static void axpy_1D( const floatType* __restrict__ A, floatType* __restrict__ B,
}

template<int betaIsZero, typename floatType, bool useStreamingStores, bool spawnThreads, bool conjA>
static void axpy_2D( const floatType* __restrict__ A, const int lda,
floatType* __restrict__ B, const int ldb,
static void axpy_2D( const floatType* HPTT_RESTRICT A, const int lda,
floatType* HPTT_RESTRICT B, const int ldb,
const int n0, const int myStart, const int myEnd, const floatType alpha, const floatType beta, int numThreads)
{
if( !betaIsZero )
Expand Down Expand Up @@ -1726,7 +1729,9 @@ void Transpose<floatType>::getBestLoopOrder( std::vector<int> &loopOrder ) const
}

// create cost matrix; cost[i,idx] === cost for idx being at loop-level i
double costs[dim_*dim_];
// dim_ is a runtime value, and variable-length arrays are a GNU
// extension that MSVC rejects (C2131).
std::vector<double> costs(dim_*dim_);
for(int i=0;i < dim_ ; ++i){
for(int idx=0;idx < dim_ ; ++idx){ //idx is at loop i
double cost = 0;
Expand Down