From 590bb3b8c5c0389159c5a9451b6c156c5f237e8a Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Sun, 24 May 2015 16:02:53 -0500 Subject: [PATCH 01/38] Backed-out adjusted dim changes to test/3m4m. Details: - Reverted most changes applied during commit ec25807b. --- test/3m4m/Makefile | 57 ++++++++++++++++++------------------------- test/3m4m/runme.sh | 4 +-- test/3m4m/test_gemm.c | 18 ++++++-------- 3 files changed, 33 insertions(+), 46 deletions(-) diff --git a/test/3m4m/Makefile b/test/3m4m/Makefile index bfe55935e5..ecc6a84ef2 100644 --- a/test/3m4m/Makefile +++ b/test/3m4m/Makefile @@ -221,9 +221,6 @@ STR_NAT := -DSTR=\"asm\" STR_OBL := -DSTR=\"openblas\" STR_MKL := -DSTR=\"mkl\" STR_ACML := -DSTR=\"acml\" -STR_AMK := -DSTR=\"amk\" -STR_AKN := -DSTR=\"akn\" -STR_AMN := -DSTR=\"amn\" # Single or multithreaded string STR_ST := -DTHR_STR=\"st\" @@ -238,10 +235,6 @@ PDEF_MT := -DP_BEGIN=400 \ -DP_END=8000 \ -DP_INC=400 -ADJ_MK := -DADJ_MK -ADJ_KN := -DADJ_KN -ADJ_MN := -DADJ_MN - # @@ -281,14 +274,7 @@ blis: \ test_cgemm_4m1a_blis_st.x \ test_zgemm_4m1a_blis_st.x \ test_cgemm_asm_blis_st.x \ - test_zgemm_asm_blis_st.x \ - \ - test_cgemm_amk_blis_st.x \ - test_zgemm_amk_blis_st.x \ - test_cgemm_akn_blis_st.x \ - test_zgemm_akn_blis_st.x \ - test_cgemm_amn_blis_st.x \ - test_zgemm_amn_blis_st.x + test_zgemm_asm_blis_st.x blis-mt: \ test_sgemm_asm_blis_mt.x \ @@ -312,6 +298,8 @@ blis-mt: \ test_zgemm_asm_blis_mt.x openblas: \ + test_sgemm_openblas_st.x \ + test_dgemm_openblas_st.x \ test_cgemm_openblas_st.x \ test_zgemm_openblas_st.x @@ -320,6 +308,8 @@ openblas-mt: \ test_zgemm_openblas_mt.x mkl: \ + test_sgemm_mkl_st.x \ + test_dgemm_mkl_st.x \ test_cgemm_mkl_st.x \ test_zgemm_mkl_st.x @@ -328,6 +318,8 @@ mkl-mt: \ test_zgemm_mkl_mt.x acml: \ + test_sgemm_acml_st.x \ + test_dgemm_acml_st.x \ test_cgemm_acml_st.x \ test_zgemm_acml_st.x @@ -458,26 +450,13 @@ test_z%_asm_blis_mt.o: test_%.c test_c%_asm_blis_mt.o: test_%.c $(CC) $(CFLAGS) $(PDEF_MT) $(DT_C) $(BLI_DEF) $(DNAT) $(STR_NAT) $(STR_MT) -c $< -o $@ -# blis asm (adjusted) -test_z%_amk_blis_st.o: test_%.c - $(CC) $(CFLAGS) $(PDEF_ST) $(DT_Z) $(BLI_DEF) $(DNAT) $(STR_AMK) $(STR_ST) $(ADJ_MK) -c $< -o $@ - -test_c%_amk_blis_st.o: test_%.c - $(CC) $(CFLAGS) $(PDEF_ST) $(DT_C) $(BLI_DEF) $(DNAT) $(STR_AMK) $(STR_ST) $(ADJ_MK) -c $< -o $@ - -test_z%_akn_blis_st.o: test_%.c - $(CC) $(CFLAGS) $(PDEF_ST) $(DT_Z) $(BLI_DEF) $(DNAT) $(STR_AKN) $(STR_ST) $(ADJ_KN) -c $< -o $@ - -test_c%_akn_blis_st.o: test_%.c - $(CC) $(CFLAGS) $(PDEF_ST) $(DT_C) $(BLI_DEF) $(DNAT) $(STR_AKN) $(STR_ST) $(ADJ_KN) -c $< -o $@ - -test_z%_amn_blis_st.o: test_%.c - $(CC) $(CFLAGS) $(PDEF_ST) $(DT_Z) $(BLI_DEF) $(DNAT) $(STR_AMN) $(STR_ST) $(ADJ_MN) -c $< -o $@ +# openblas +test_d%_openblas_st.o: test_%.c + $(CC) $(CFLAGS) $(PDEF_ST) $(DT_D) $(BLA_DEF) $(DNAT) $(STR_OBL) $(STR_ST) -c $< -o $@ -test_c%_amn_blis_st.o: test_%.c - $(CC) $(CFLAGS) $(PDEF_ST) $(DT_C) $(BLI_DEF) $(DNAT) $(STR_AMN) $(STR_ST) $(ADJ_MN) -c $< -o $@ +test_s%_openblas_st.o: test_%.c + $(CC) $(CFLAGS) $(PDEF_ST) $(DT_S) $(BLA_DEF) $(DNAT) $(STR_OBL) $(STR_ST) -c $< -o $@ -# openblas test_z%_openblas_st.o: test_%.c $(CC) $(CFLAGS) $(PDEF_ST) $(DT_Z) $(BLA_DEF) $(DNAT) $(STR_OBL) $(STR_ST) -c $< -o $@ @@ -491,6 +470,12 @@ test_c%_openblas_mt.o: test_%.c $(CC) $(CFLAGS) $(PDEF_MT) $(DT_C) $(BLA_DEF) $(DNAT) $(STR_OBL) $(STR_MT) -c $< -o $@ # mkl +test_d%_mkl_st.o: test_%.c + $(CC) $(CFLAGS) $(PDEF_ST) $(DT_D) $(BLA_DEF) $(DNAT) $(STR_MKL) $(STR_ST) -c $< -o $@ + +test_s%_mkl_st.o: test_%.c + $(CC) $(CFLAGS) $(PDEF_ST) $(DT_S) $(BLA_DEF) $(DNAT) $(STR_MKL) $(STR_ST) -c $< -o $@ + test_z%_mkl_st.o: test_%.c $(CC) $(CFLAGS) $(PDEF_ST) $(DT_Z) $(BLA_DEF) $(DNAT) $(STR_MKL) $(STR_ST) -c $< -o $@ @@ -504,6 +489,12 @@ test_c%_mkl_mt.o: test_%.c $(CC) $(CFLAGS) $(PDEF_MT) $(DT_C) $(BLA_DEF) $(DNAT) $(STR_MKL) $(STR_MT) -c $< -o $@ # acml +test_d%_acml_st.o: test_%.c + $(CC) $(CFLAGS) $(PDEF_ST) $(DT_D) $(BLA_DEF) $(DNAT) $(STR_ACML) $(STR_ST) -c $< -o $@ + +test_s%_acml_st.o: test_%.c + $(CC) $(CFLAGS) $(PDEF_ST) $(DT_S) $(BLA_DEF) $(DNAT) $(STR_ACML) $(STR_ST) -c $< -o $@ + test_z%_acml_st.o: test_%.c $(CC) $(CFLAGS) $(PDEF_ST) $(DT_Z) $(BLA_DEF) $(DNAT) $(STR_ACML) $(STR_ST) -c $< -o $@ diff --git a/test/3m4m/runme.sh b/test/3m4m/runme.sh index 2333b03678..9bd11f02a7 100755 --- a/test/3m4m/runme.sh +++ b/test/3m4m/runme.sh @@ -4,8 +4,8 @@ exec_root="test" out_root="output" -sys="stampede" -#sys="wahlberg" +#sys="stampede" +sys="wahlberg" # Bind threads to processors. #export OMP_PROC_BIND=true diff --git a/test/3m4m/test_gemm.c b/test/3m4m/test_gemm.c index afc2209bf9..0a45266d92 100644 --- a/test/3m4m/test_gemm.c +++ b/test/3m4m/test_gemm.c @@ -49,7 +49,7 @@ int main( int argc, char** argv ) dim_t p; dim_t p_begin, p_end, p_inc; int m_input, n_input, k_input; - num_t dt, dt_real; + num_t dt; char dt_ch; int r, n_repeats; trans_t transa; @@ -61,7 +61,6 @@ int main( int argc, char** argv ) double dtime_save; double gflops; - extern blksz_t* gemm_kc; bli_init(); @@ -70,7 +69,6 @@ int main( int argc, char** argv ) n_repeats = 3; dt = DT; - dt_real = bli_datatype_proj_to_real( DT ); p_begin = P_BEGIN; p_end = P_END; @@ -80,6 +78,11 @@ int main( int argc, char** argv ) n_input = -1; k_input = -1; +#if 0 + extern blksz_t* gemm_kc; + + num_t dt_real = bli_datatype_proj_to_real( DT ); + // Extract the kc blocksize for the requested datatype and its // real analogue. dim_t kc = bli_blksz_get_def( dt, gemm_kc ); @@ -92,14 +95,6 @@ int main( int argc, char** argv ) else if ( IND == BLIS_3M1 ) k_input = kc_real / 3; else if ( IND == BLIS_4M1A ) k_input = kc_real / 2; else k_input = kc_real; - - // Adjust the relative dimensions, if requested. -#if (defined ADJ_MK) - m_input = -2; k_input = -2; n_input = -1; -#elif (defined ADJ_KN) - k_input = -2; n_input = -2; m_input = -1; -#elif (defined ADJ_MN) - m_input = -2; n_input = -2; k_input = -1; #endif // Choose the char corresponding to the requested datatype. @@ -131,6 +126,7 @@ int main( int argc, char** argv ) for ( p = p_begin; p <= p_end; p += p_inc ) { + if ( m_input < 0 ) m = p / ( dim_t )abs(m_input); else m = ( dim_t ) m_input; if ( n_input < 0 ) n = p / ( dim_t )abs(n_input); From 1213f5cebabc1637ce9dd45c4bfa87bb93677c29 Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Tue, 2 Jun 2015 13:27:47 -0500 Subject: [PATCH 02/38] POSIX thread bugfixes/edits to bli_init.c, _mem.c. Details: - Fixed a sort-of bug in bli_init.c whereby the wrong pthread mutex was used to lock access to initialization/finalization actions. But everything worked out okay as long as bli_init() was called by single-threaded code. - Changed to static initialization for memory allocator mutex in bli_mem.c, and moved mutex to that file (from bli_init.c). - Fixed some type mismatches in bli_threading_pthreads.c that resulted in compiler warnings. - Fixed a small memory leak with allocated-but-never-freed (and unused) pthread_attr_t objects. - Whitespace changes to bli_init.c and bli_mem.c. --- frame/base/bli_init.c | 22 ++++----- frame/base/bli_mem.c | 75 ++++++++++++++++++++++------- frame/base/bli_threading_pthreads.c | 11 ++++- 3 files changed, 75 insertions(+), 33 deletions(-) diff --git a/frame/base/bli_init.c b/frame/base/bli_init.c index ae7e8ba280..5ef8754c58 100644 --- a/frame/base/bli_init.c +++ b/frame/base/bli_init.c @@ -53,7 +53,6 @@ thread_comm_t BLIS_SINGLE_COMM; #ifdef BLIS_ENABLE_PTHREADS pthread_mutex_t initialize_mutex = PTHREAD_MUTEX_INITIALIZER; -pthread_mutex_t mem_manager_mutex; #endif err_t bli_init( void ) @@ -79,11 +78,11 @@ err_t bli_init( void ) // section to prevent a race condition of the type described above. // BEGIN CRITICAL SECTION -#ifdef BLIS_ENABLE_OPENMP +#ifdef BLIS_ENABLE_OPENMP _Pragma( "omp critical (init)" ) #endif #ifdef BLIS_ENABLE_PTHREADS - pthread_mutex_lock( &mem_manager_mutex ); + pthread_mutex_lock( &initialize_mutex ); #endif { @@ -102,9 +101,9 @@ err_t bli_init( void ) bli_error_msgs_init(); bli_mem_init(); - + bli_ind_init(); - + bli_setup_communicator( &BLIS_SINGLE_COMM, 1 ); bli_setup_packm_single_threaded_info( &BLIS_PACKM_SINGLE_THREADED ); bli_setup_gemm_single_threaded_info( &BLIS_GEMM_SINGLE_THREADED ); @@ -121,12 +120,9 @@ err_t bli_init( void ) // END CRITICAL SECTION } #ifdef BLIS_ENABLE_PTHREADS - pthread_mutex_unlock( &mem_manager_mutex ); + pthread_mutex_unlock( &initialize_mutex ); #endif -#ifdef BLIS_ENABLE_PTHREADS - pthread_mutex_init( &mem_manager_mutex, NULL ); -#endif return r_val; } @@ -153,11 +149,11 @@ err_t bli_finalize( void ) // section to prevent a race condition of the type described above. // BEGIN CRITICAL SECTION -#ifdef BLIS_ENABLE_OPENMP +#ifdef BLIS_ENABLE_OPENMP _Pragma( "omp critical (init)" ) #endif #ifdef BLIS_ENABLE_PTHREADS - pthread_mutex_lock( &mem_manager_mutex ); + pthread_mutex_lock( &initialize_mutex ); #endif { @@ -188,11 +184,11 @@ err_t bli_finalize( void ) // END CRITICAL SECTION } #ifdef BLIS_ENABLE_PTHREADS - pthread_mutex_unlock( &mem_manager_mutex ); + pthread_mutex_unlock( &initialize_mutex ); #endif #ifdef BLIS_ENABLE_PTHREADS - pthread_mutex_destroy( &mem_manager_mutex ); + pthread_mutex_destroy( &initialize_mutex ); #endif return r_val; diff --git a/frame/base/bli_mem.c b/frame/base/bli_mem.c index e789f3f1dd..cd5717e310 100644 --- a/frame/base/bli_mem.c +++ b/frame/base/bli_mem.c @@ -35,7 +35,7 @@ #include "blis.h" #ifdef BLIS_ENABLE_PTHREADS -extern pthread_mutex_t mem_manager_mutex; +pthread_mutex_t mem_manager_mutex = PTHREAD_MUTEX_INITIALIZER; #endif // Declare one memory pool structure for each block size/shape we want to @@ -128,14 +128,14 @@ void bli_mem_acquire_m( siz_t req_size, block_ptrs = bli_pool_block_ptrs( pool ); - // BEGIN CRITICAL SECTION -#ifdef BLIS_ENABLE_OPENMP - _Pragma( "omp critical (mem)" ) +#ifdef BLIS_ENABLE_OPENMP + _Pragma( "omp critical (mem)" ) #endif #ifdef BLIS_ENABLE_PTHREADS - pthread_mutex_lock( &mem_manager_mutex ); + pthread_mutex_lock( &mem_manager_mutex ); #endif - { + // BEGIN CRITICAL SECTION + { // Query the index of the contiguous memory block that resides at the // "top" of the pool. @@ -152,10 +152,11 @@ void bli_mem_acquire_m( siz_t req_size, bli_pool_dec_top_index( pool ); + } // END CRITICAL SECTION - } + #ifdef BLIS_ENABLE_PTHREADS - pthread_mutex_unlock( &mem_manager_mutex ); + pthread_mutex_unlock( &mem_manager_mutex ); #endif // Query the size of the blocks in the pool so we can store it in the @@ -208,15 +209,15 @@ void bli_mem_release( mem_t* mem ) // Extract the block pointer array associated with the pool. block_ptrs = bli_pool_block_ptrs( pool ); - - // BEGIN CRITICAL SECTION -#ifdef BLIS_ENABLE_OPENMP - _Pragma( "omp critical (mem)" ) +#ifdef BLIS_ENABLE_OPENMP + _Pragma( "omp critical (mem)" ) #endif #ifdef BLIS_ENABLE_PTHREADS - pthread_mutex_lock( &mem_manager_mutex ); + pthread_mutex_lock( &mem_manager_mutex ); #endif - { + + // BEGIN CRITICAL SECTION + { // Increment the top of the memory pool. bli_pool_inc_top_index( pool ); @@ -227,11 +228,11 @@ void bli_mem_release( mem_t* mem ) // Place the address of the block back onto the top of the memory pool. block_ptrs[i] = block; - + } // END CRITICAL SECTION - } + #ifdef BLIS_ENABLE_PTHREADS - pthread_mutex_unlock( &mem_manager_mutex ); + pthread_mutex_unlock( &mem_manager_mutex ); #endif } @@ -269,6 +270,16 @@ void bli_mem_init() index_b = bli_packbuf_index( BLIS_BUFFER_FOR_B_PANEL ); index_c = bli_packbuf_index( BLIS_BUFFER_FOR_C_PANEL ); +#ifdef BLIS_ENABLE_OPENMP + _Pragma( "omp critical (mem)" ) +#endif +#ifdef BLIS_ENABLE_PTHREADS + pthread_mutex_lock( &mem_manager_mutex ); +#endif + + // BEGIN CRITICAL SECTION + { + // Initialize contiguous memory pool for MC x KC blocks. bli_mem_init_pool( pool_mk_mem, BLIS_MK_BLOCK_SIZE, @@ -289,6 +300,13 @@ void bli_mem_init() BLIS_NUM_MC_X_NC_BLOCKS, pool_mn_blk_ptrs, &pools[ index_c ] ); + + } + // END CRITICAL SECTION + +#ifdef BLIS_ENABLE_PTHREADS + pthread_mutex_unlock( &mem_manager_mutex ); +#endif } @@ -343,6 +361,27 @@ void bli_mem_init_pool( char* pool_mem, void bli_mem_finalize() { - // Nothing to do. + +#ifdef BLIS_ENABLE_OPENMP + _Pragma( "omp critical (mem)" ) +#endif +#ifdef BLIS_ENABLE_PTHREADS + pthread_mutex_lock( &mem_manager_mutex ); +#endif + + // BEGIN CRITICAL SECTION + { + // Do nothing. + } + // END CRITICAL SECTION + +#ifdef BLIS_ENABLE_PTHREADS + pthread_mutex_unlock( &mem_manager_mutex ); +#endif + +#ifdef BLIS_ENABLE_PTHREADS + pthread_mutex_destroy( &mem_manager_mutex ); +#endif + } diff --git a/frame/base/bli_threading_pthreads.c b/frame/base/bli_threading_pthreads.c index caf8a8c005..0aad78c2aa 100644 --- a/frame/base/bli_threading_pthreads.c +++ b/frame/base/bli_threading_pthreads.c @@ -36,6 +36,8 @@ #ifdef BLIS_ENABLE_PTHREADS +void* thread_decorator_helper( void* data_void ); + typedef struct thread_data { level3_int_t func; @@ -48,10 +50,15 @@ typedef struct thread_data void* thread; } thread_data_t; -void* thread_decorator_helper( thread_data_t* data ) +void* thread_decorator_helper( void* data_void ) { + thread_data_t* data = data_void; + data->func( data->alpha, data->a, data->b, data->beta, data->c, data->cntl, data->thread ); + + return NULL; } + void bli_level3_thread_decorator( dim_t n_threads, level3_int_t func, obj_t* alpha, @@ -65,7 +72,7 @@ void bli_level3_thread_decorator( dim_t n_threads, pthread_t* pthreads = (pthread_t*) bli_malloc(sizeof(pthread_t) * n_threads); //Saying "datas" is kind of like saying "all y'all" thread_data_t* datas = (thread_data_t*) bli_malloc(sizeof(thread_data_t) * n_threads); - pthread_attr_t* attr = (pthread_attr_t*) bli_malloc(sizeof(pthread_attr_t) * n_threads); + //pthread_attr_t* attr = (pthread_attr_t*) bli_malloc(sizeof(pthread_attr_t) * n_threads); for( int i = 0; i < n_threads; i++ ) { From b6ee82a3d421c9c4f1eb6848c7c6e37aa46de799 Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Wed, 3 Jun 2015 12:14:23 -0500 Subject: [PATCH 03/38] Minor cleanup to bli_init() and friends. Details: - Spun-off initialization of global scalar constants to bli_const_init() and of threading stuff to bli_thread_init(). - Added some missing _finalize() functions, even when there is nothing to do. --- frame/base/bli_const.c | 69 +++++++++++++++++++++++++++++++++ frame/base/bli_const.h | 37 ++++++++++++++++++ frame/base/bli_error.c | 7 +++- frame/base/bli_error.h | 3 +- frame/base/bli_init.c | 67 ++++++-------------------------- frame/base/bli_init.h | 3 -- frame/base/bli_threading.c | 19 +++++++++ frame/base/bli_threading.h | 3 ++ frame/include/blis.h | 1 + frame/ind/query/bli_ind_query.c | 5 +++ frame/ind/query/bli_ind_query.h | 1 + 11 files changed, 155 insertions(+), 60 deletions(-) create mode 100644 frame/base/bli_const.c create mode 100644 frame/base/bli_const.h diff --git a/frame/base/bli_const.c b/frame/base/bli_const.c new file mode 100644 index 0000000000..316f115a45 --- /dev/null +++ b/frame/base/bli_const.c @@ -0,0 +1,69 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + +// These globally-declared obj_t's are used to store the so-called +// "global scalar constants" for commonly-used 1x1 values. + +obj_t BLIS_TWO; +obj_t BLIS_ONE; +obj_t BLIS_ONE_HALF; +obj_t BLIS_ZERO; +obj_t BLIS_MINUS_ONE_HALF; +obj_t BLIS_MINUS_ONE; +obj_t BLIS_MINUS_TWO; + +void bli_const_init( void ) +{ + bli_obj_create_const( 2.0, &BLIS_TWO ); + bli_obj_create_const( 1.0, &BLIS_ONE ); + bli_obj_create_const( 0.5, &BLIS_ONE_HALF ); + bli_obj_create_const( 0.0, &BLIS_ZERO ); + bli_obj_create_const( -0.5, &BLIS_MINUS_ONE_HALF ); + bli_obj_create_const( -1.0, &BLIS_MINUS_ONE ); + bli_obj_create_const( -2.0, &BLIS_MINUS_TWO ); +} + +void bli_const_finalize( void ) +{ + bli_obj_free( &BLIS_TWO ); + bli_obj_free( &BLIS_ONE ); + bli_obj_free( &BLIS_ONE_HALF ); + bli_obj_free( &BLIS_ZERO ); + bli_obj_free( &BLIS_MINUS_ONE_HALF ); + bli_obj_free( &BLIS_MINUS_ONE ); + bli_obj_free( &BLIS_MINUS_TWO ); +} + diff --git a/frame/base/bli_const.h b/frame/base/bli_const.h new file mode 100644 index 0000000000..066fd5e72a --- /dev/null +++ b/frame/base/bli_const.h @@ -0,0 +1,37 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +void bli_const_init( void ); +void bli_const_finalize( void ); + diff --git a/frame/base/bli_error.c b/frame/base/bli_error.c index 47634609ce..11064a72ac 100644 --- a/frame/base/bli_error.c +++ b/frame/base/bli_error.c @@ -86,7 +86,7 @@ void bli_print_msg( char* str, char* file, guint_t line ) fflush( stderr ); } -void bli_error_msgs_init( void ) +void bli_error_init( void ) { sprintf( bli_error_string_for_code(BLIS_INVALID_ERROR_CHECKING_LEVEL), "Invalid error checking level." ); @@ -196,3 +196,8 @@ void bli_error_msgs_init( void ) "Expected object to be alias." ); } +void bli_error_finalize( void ) +{ + // Nothing to do. +} + diff --git a/frame/base/bli_error.h b/frame/base/bli_error.h index acba60a730..6f16472b08 100644 --- a/frame/base/bli_error.h +++ b/frame/base/bli_error.h @@ -43,5 +43,6 @@ char* bli_error_string_for_code( gint_t code ); void bli_abort( void ); void bli_print_msg( char* str, char* file, guint_t line ); -void bli_error_msgs_init( void ); +void bli_error_init( void ); +void bli_error_finalize( void ); diff --git a/frame/base/bli_init.c b/frame/base/bli_init.c index 5ef8754c58..b69e8086d0 100644 --- a/frame/base/bli_init.c +++ b/frame/base/bli_init.c @@ -34,27 +34,13 @@ #include "blis.h" -// -- Global variables -- - -static bool_t bli_initialized = FALSE; - -obj_t BLIS_TWO; -obj_t BLIS_ONE; -obj_t BLIS_ONE_HALF; -obj_t BLIS_ZERO; -obj_t BLIS_MINUS_ONE_HALF; -obj_t BLIS_MINUS_ONE; -obj_t BLIS_MINUS_TWO; - -packm_thrinfo_t BLIS_PACKM_SINGLE_THREADED; -gemm_thrinfo_t BLIS_GEMM_SINGLE_THREADED; -herk_thrinfo_t BLIS_HERK_SINGLE_THREADED; -thread_comm_t BLIS_SINGLE_COMM; - #ifdef BLIS_ENABLE_PTHREADS pthread_mutex_t initialize_mutex = PTHREAD_MUTEX_INITIALIZER; #endif +static bool_t bli_initialized = FALSE; + + err_t bli_init( void ) { err_t r_val = BLIS_FAILURE; @@ -94,20 +80,13 @@ err_t bli_init( void ) // necessary initialization subtasks). if ( bli_initialized == FALSE ) { - bli_init_const(); - + // Initialize various sub-APIs. + bli_const_init(); bli_cntl_init(); - - bli_error_msgs_init(); - + bli_error_init(); bli_mem_init(); - bli_ind_init(); - - bli_setup_communicator( &BLIS_SINGLE_COMM, 1 ); - bli_setup_packm_single_threaded_info( &BLIS_PACKM_SINGLE_THREADED ); - bli_setup_gemm_single_threaded_info( &BLIS_GEMM_SINGLE_THREADED ); - bli_setup_herk_single_threaded_info( &BLIS_HERK_SINGLE_THREADED ); + bli_thread_init(); // After initialization is complete, mark BLIS as initialized. bli_initialized = TRUE; @@ -165,13 +144,13 @@ err_t bli_finalize( void ) // necessary finalization subtasks). if ( bli_initialized == TRUE ) { - bli_finalize_const(); - + // Finalize various sub-APIs. + bli_const_finalize(); bli_cntl_finalize(); - - // Don't need to do anything to finalize error messages. - + bli_error_finalize(); bli_mem_finalize(); + bli_ind_finalize(); + bli_thread_finalize(); // After finalization is complete, mark BLIS as uninitialized. bli_initialized = FALSE; @@ -194,28 +173,6 @@ err_t bli_finalize( void ) return r_val; } -void bli_init_const( void ) -{ - bli_obj_create_const( 2.0, &BLIS_TWO ); - bli_obj_create_const( 1.0, &BLIS_ONE ); - bli_obj_create_const( 0.5, &BLIS_ONE_HALF ); - bli_obj_create_const( 0.0, &BLIS_ZERO ); - bli_obj_create_const( -0.5, &BLIS_MINUS_ONE_HALF ); - bli_obj_create_const( -1.0, &BLIS_MINUS_ONE ); - bli_obj_create_const( -2.0, &BLIS_MINUS_TWO ); -} - -void bli_finalize_const( void ) -{ - bli_obj_free( &BLIS_TWO ); - bli_obj_free( &BLIS_ONE ); - bli_obj_free( &BLIS_ONE_HALF ); - bli_obj_free( &BLIS_ZERO ); - bli_obj_free( &BLIS_MINUS_ONE_HALF ); - bli_obj_free( &BLIS_MINUS_ONE ); - bli_obj_free( &BLIS_MINUS_TWO ); -} - void bli_init_auto( err_t* init_result ) { *init_result = bli_init(); diff --git a/frame/base/bli_init.h b/frame/base/bli_init.h index 670d776c8a..cd620008df 100644 --- a/frame/base/bli_init.h +++ b/frame/base/bli_init.h @@ -35,8 +35,5 @@ err_t bli_init( void ); err_t bli_finalize( void ); -void bli_init_const( void ); -void bli_finalize_const( void ); - void bli_init_auto( err_t* init_result ); void bli_finalize_auto( err_t init_result ); diff --git a/frame/base/bli_threading.c b/frame/base/bli_threading.c index 65ef8e6a36..f6f0a241ce 100644 --- a/frame/base/bli_threading.c +++ b/frame/base/bli_threading.c @@ -34,6 +34,25 @@ #include "blis.h" +packm_thrinfo_t BLIS_PACKM_SINGLE_THREADED; +gemm_thrinfo_t BLIS_GEMM_SINGLE_THREADED; +herk_thrinfo_t BLIS_HERK_SINGLE_THREADED; +thread_comm_t BLIS_SINGLE_COMM; + +void bli_thread_init( void ) +{ + + bli_setup_communicator( &BLIS_SINGLE_COMM, 1 ); + bli_setup_packm_single_threaded_info( &BLIS_PACKM_SINGLE_THREADED ); + bli_setup_gemm_single_threaded_info( &BLIS_GEMM_SINGLE_THREADED ); + bli_setup_herk_single_threaded_info( &BLIS_HERK_SINGLE_THREADED ); +} + +void bli_thread_finalize( void ) +{ + // Nothing to do. +} + //*********** Stuff Specific to single-threaded ************* #ifndef BLIS_ENABLE_MULTITHREADING void bli_barrier( thread_comm_t* communicator, dim_t t_id ) diff --git a/frame/base/bli_threading.h b/frame/base/bli_threading.h index 923e16509b..ce4b21c553 100644 --- a/frame/base/bli_threading.h +++ b/frame/base/bli_threading.h @@ -87,6 +87,9 @@ typedef struct thread_comm_s thread_comm_t; #include "bli_threading_omp.h" #include "bli_threading_pthreads.h" +void bli_thread_init( void ); +void bli_thread_finalize( void ); + // Thread Communicator Interface Definitions void bli_setup_communicator( thread_comm_t* communicator, dim_t n_threads ); void bli_cleanup_communicator( thread_comm_t* communicator ); diff --git a/frame/include/blis.h b/frame/include/blis.h index 59efad5755..045cdc894d 100644 --- a/frame/include/blis.h +++ b/frame/include/blis.h @@ -104,6 +104,7 @@ extern "C" { // -- Base operation prototypes -- #include "bli_init.h" +#include "bli_const.h" #include "bli_malloc.h" #include "bli_obj.h" #include "bli_obj_scalar.h" diff --git a/frame/ind/query/bli_ind_query.c b/frame/ind/query/bli_ind_query.c index 6e1e19c3a0..c48e1a6859 100644 --- a/frame/ind/query/bli_ind_query.c +++ b/frame/ind/query/bli_ind_query.c @@ -222,6 +222,11 @@ void bli_ind_init( void ) #endif } +void bli_ind_finalize( void ) +{ + // Nothing to do. +} + // ----------------------------------------------------------------------------- void bli_ind_enable( ind_t method ) diff --git a/frame/ind/query/bli_ind_query.h b/frame/ind/query/bli_ind_query.h index ff917389b0..4b199ff8e0 100644 --- a/frame/ind/query/bli_ind_query.h +++ b/frame/ind/query/bli_ind_query.h @@ -83,6 +83,7 @@ ind_t bli_ind_oper_find_avail( opid_t oper, num_t dt ); char* bli_ind_oper_get_avail_impl_string( opid_t oper, num_t dt ); void bli_ind_init( void ); +void bli_ind_finalize( void ); void bli_ind_enable( ind_t method ); void bli_ind_disable( ind_t method ); From d62ceece943b20537ec4dd99f25136b9ba2ae340 Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Wed, 3 Jun 2015 12:56:45 -0500 Subject: [PATCH 04/38] Minor update to test/3m4m/runme.sh. Details: - Removed some stale script code that should have been removed during 590bb3b8c. --- test/3m4m/runme.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/3m4m/runme.sh b/test/3m4m/runme.sh index 9bd11f02a7..5aa12a2daf 100755 --- a/test/3m4m/runme.sh +++ b/test/3m4m/runme.sh @@ -66,9 +66,9 @@ elif [ ${sys} = "wahlberg" ]; then test_impls="openblas acml asm_blis 3mhw_blis 3m3_blis 3m2_blis 3m1_blis 4mhw_blis 4m1b_blis 4m1a_blis" #test_impls="openblas acml asm_blis" fi -test_impls="asm_blis amk_blis akn_blis amn_blis" # Real domain implementations to test. +#test_impls_r="openblas acml asm_blis" test_impls_r="asm_blis" # First perform real test cases. From 9135dfd69d39f3bbd75034f479f27a78dbfebcce Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Fri, 5 Jun 2015 13:37:44 -0500 Subject: [PATCH 05/38] Minor updates to test/3m4m files. --- test/3m4m/Makefile | 32 ++++++++++++++++++-------------- test/3m4m/runme.sh | 5 ++--- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/test/3m4m/Makefile b/test/3m4m/Makefile index ecc6a84ef2..e83ca4bc14 100644 --- a/test/3m4m/Makefile +++ b/test/3m4m/Makefile @@ -45,8 +45,8 @@ # .PHONY: all \ - blis openblas mkl \ - blis-mt openblas-mt mkl-mt \ + blis-gemm-st openblas-gemm-st mkl-gemm-st acml-gemm-st \ + blis-gemm-mt openblas-gemm-mt mkl-gemm-mt acml-gemm-mt \ clean cleanx @@ -247,15 +247,19 @@ intel: blis-all openblas-all mkl-all amd: blis-all openblas-all acml-all -blis-all: blis blis-mt +blis-all: blis-gemm-st \ + blis-gemm-mt -openblas-all: openblas openblas-mt +openblas-all: openblas-gemm-st \ + openblas-gemm-mt -mkl-all: mkl mkl-mt +mkl-all: mkl-gemm-st \ + mkl-gemm-mt -acml-all: acml acml-mt +acml-all: acml-gemm-st \ + acml-gemm-mt -blis: \ +blis-gemm-st: \ test_sgemm_asm_blis_st.x \ test_dgemm_asm_blis_st.x \ \ @@ -276,7 +280,7 @@ blis: \ test_cgemm_asm_blis_st.x \ test_zgemm_asm_blis_st.x -blis-mt: \ +blis-gemm-mt: \ test_sgemm_asm_blis_mt.x \ test_dgemm_asm_blis_mt.x \ \ @@ -297,33 +301,33 @@ blis-mt: \ test_cgemm_asm_blis_mt.x \ test_zgemm_asm_blis_mt.x -openblas: \ +openblas-gemm-st: \ test_sgemm_openblas_st.x \ test_dgemm_openblas_st.x \ test_cgemm_openblas_st.x \ test_zgemm_openblas_st.x -openblas-mt: \ +openblas-gemm-mt: \ test_cgemm_openblas_mt.x \ test_zgemm_openblas_mt.x -mkl: \ +mkl-gemm-st: \ test_sgemm_mkl_st.x \ test_dgemm_mkl_st.x \ test_cgemm_mkl_st.x \ test_zgemm_mkl_st.x -mkl-mt: \ +mkl-gemm-mt: \ test_cgemm_mkl_mt.x \ test_zgemm_mkl_mt.x -acml: \ +acml-gemm-st: \ test_sgemm_acml_st.x \ test_dgemm_acml_st.x \ test_cgemm_acml_st.x \ test_zgemm_acml_st.x -acml-mt: \ +acml-gemm-mt: \ test_cgemm_acml_mt.x \ test_zgemm_acml_mt.x diff --git a/test/3m4m/runme.sh b/test/3m4m/runme.sh index 5aa12a2daf..445e40bd0b 100755 --- a/test/3m4m/runme.sh +++ b/test/3m4m/runme.sh @@ -64,12 +64,11 @@ if [ ${sys} = "stampede" ]; then elif [ ${sys} = "wahlberg" ]; then test_impls="openblas acml asm_blis 3mhw_blis 3m3_blis 3m2_blis 3m1_blis 4mhw_blis 4m1b_blis 4m1a_blis" - #test_impls="openblas acml asm_blis" + test_impls="openblas acml asm_blis" fi # Real domain implementations to test. -#test_impls_r="openblas acml asm_blis" -test_impls_r="asm_blis" +test_impls_r="openblas acml asm_blis" # First perform real test cases. for th in ${threads_r}; do From ee129c6b028bc5ac88da7c74fde72c49803742ff Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Wed, 10 Jun 2015 12:53:28 -0500 Subject: [PATCH 06/38] Fixed bugs in _get_range(), _get_range_weighted(). Details: - Fixed some bugs that only manifested in multithreaded instances of some (non-gemm) level-3 operations. The bugs were related to invalid allocation of "edge" cases to thread subpartitions. (Here, we define an "edge" case to be one where the dimension being partitioned for parallelism is not a whole multiple of whatever register blocksize is needed in that dimension.) In BLIS, we always require edge cases to be part of the bottom, right, or bottom-right subpartitions. (This is so that zero-padding only has to happen at the bottom, right, or bottom-right edges of micro-panels.) The previous implementations of bli_get_range() and _get_range_weighted() did not adhere to this implicit policy and thus produced bad ranges for some combinations of operation, parameter cases, problem sizes, and n-way parallelism. - As part of the above fix, the functions bli_get_range() and _get_range_weighted() have been renamed to use _l2r, _r2l, _t2b, and _b2t suffixes, similar to the partitioning functions. This is an easy way to make sure that the variants are calling the right version of each function. The function signatures have also been changed slightly. - Comment/whitespace updates. - Removed unnecessary '/' from macros in bli_obj_macro_defs.h. --- frame/3/gemm/bli_gemm_blk_var1f.c | 10 +- frame/3/gemm/bli_gemm_blk_var2f.c | 10 +- frame/3/gemm/ind/bli_gemm_blk_var4f.c | 14 +- frame/3/herk/bli_herk_blk_var1f.c | 12 +- frame/3/herk/bli_herk_blk_var2f.c | 6 +- frame/3/trmm/bli_trmm_blk_var1f.c | 6 +- frame/3/trmm/bli_trmm_blk_var2b.c | 6 +- frame/3/trmm/bli_trmm_blk_var2f.c | 6 +- frame/3/trmm3/bli_trmm3_front.c | 5 +- frame/3/trsm/bli_trsm_blk_var1b.c | 8 +- frame/3/trsm/bli_trsm_blk_var1f.c | 8 +- frame/3/trsm/bli_trsm_blk_var2b.c | 12 +- frame/3/trsm/bli_trsm_blk_var2f.c | 12 +- frame/3/trsm/bli_trsm_blk_var3b.c | 4 +- frame/3/trsm/bli_trsm_blk_var3f.c | 4 +- frame/base/bli_threading.c | 372 ++++++++++++++++++++++---- frame/base/bli_threading.h | 36 ++- frame/include/bli_obj_macro_defs.h | 20 +- 18 files changed, 421 insertions(+), 130 deletions(-) diff --git a/frame/3/gemm/bli_gemm_blk_var1f.c b/frame/3/gemm/bli_gemm_blk_var1f.c index f469154745..535cc4710c 100644 --- a/frame/3/gemm/bli_gemm_blk_var1f.c +++ b/frame/3/gemm/bli_gemm_blk_var1f.c @@ -44,7 +44,7 @@ void bli_gemm_blk_var1f( obj_t* a, obj_t b_pack_s; obj_t a1_pack_s, c1_pack_s; - obj_t a1, c1; + obj_t a1, c1; obj_t* a1_pack = NULL; obj_t* b_pack = NULL; obj_t* c1_pack = NULL; @@ -83,9 +83,9 @@ void bli_gemm_blk_var1f( obj_t* a, // Query dimension in partitioning direction. m_trans = bli_obj_length_after_trans( *a ); dim_t start, end; - bli_get_range( thread, 0, m_trans, - bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), - &start, &end ); + bli_get_range_t2b( thread, 0, m_trans, + bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), + &start, &end ); // Partition along the m dimension. for ( i = start; i < end; i += b_alg ) @@ -130,7 +130,7 @@ void bli_gemm_blk_var1f( obj_t* a, c1_pack, cntl_sub_gemm( cntl ), gemm_thread_sub_gemm( thread ) ); - + thread_ibarrier( thread ); // Unpack C1 (if C1 was packed). diff --git a/frame/3/gemm/bli_gemm_blk_var2f.c b/frame/3/gemm/bli_gemm_blk_var2f.c index b13f21778b..f122f9a710 100644 --- a/frame/3/gemm/bli_gemm_blk_var2f.c +++ b/frame/3/gemm/bli_gemm_blk_var2f.c @@ -42,7 +42,7 @@ void bli_gemm_blk_var2f( obj_t* a, { obj_t a_pack_s; obj_t b1_pack_s, c1_pack_s; - + obj_t b1, c1; obj_t* a_pack = NULL; obj_t* b1_pack = NULL; @@ -82,9 +82,9 @@ void bli_gemm_blk_var2f( obj_t* a, // Query dimension in partitioning direction. n_trans = bli_obj_width_after_trans( *b ); dim_t start, end; - bli_get_range( thread, 0, n_trans, - bli_blksz_get_mult_for_obj( b, cntl_blocksize( cntl ) ), - &start, &end ); + bli_get_range_l2r( thread, 0, n_trans, + bli_blksz_get_mult_for_obj( b, cntl_blocksize( cntl ) ), + &start, &end ); // Partition along the n dimension. for ( i = start; i < end; i += b_alg ) @@ -129,7 +129,7 @@ void bli_gemm_blk_var2f( obj_t* a, c1_pack, cntl_sub_gemm( cntl ), gemm_thread_sub_gemm( thread ) ); - + thread_ibarrier( thread ); // Unpack C1 (if C1 was packed). diff --git a/frame/3/gemm/ind/bli_gemm_blk_var4f.c b/frame/3/gemm/ind/bli_gemm_blk_var4f.c index 97065c634d..d57c8c82fa 100644 --- a/frame/3/gemm/ind/bli_gemm_blk_var4f.c +++ b/frame/3/gemm/ind/bli_gemm_blk_var4f.c @@ -52,7 +52,7 @@ void bli_gemm_blk_var4f( obj_t* a, obj_t b_pack_s; obj_t a1_pack_s, c1_pack_s; - obj_t a1, c1; + obj_t a1, c1; obj_t* a1_pack = NULL; obj_t* b_pack = NULL; obj_t* c1_pack = NULL; @@ -91,9 +91,9 @@ void bli_gemm_blk_var4f( obj_t* a, // Query dimension in partitioning direction. m_trans = bli_obj_length_after_trans( *a ); dim_t start, end; - bli_get_range( thread, 0, m_trans, - bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), - &start, &end ); + bli_get_range_t2b( thread, 0, m_trans, + bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), + &start, &end ); // Partition along the m dimension. for ( i = start; i < end; i += b_alg ) @@ -140,7 +140,7 @@ void bli_gemm_blk_var4f( obj_t* a, c1_pack, cntl_sub_gemm( cntl ), gemm_thread_sub_gemm( thread ) ); - + thread_ibarrier( thread ); // Only apply beta within the first of three subproblems. @@ -167,7 +167,7 @@ void bli_gemm_blk_var4f( obj_t* a, c1_pack, cntl_sub_gemm( cntl ), gemm_thread_sub_gemm( thread ) ); - + thread_ibarrier( thread ); @@ -191,7 +191,7 @@ void bli_gemm_blk_var4f( obj_t* a, c1_pack, cntl_sub_gemm( cntl ), gemm_thread_sub_gemm( thread ) ); - + thread_ibarrier( thread ); diff --git a/frame/3/herk/bli_herk_blk_var1f.c b/frame/3/herk/bli_herk_blk_var1f.c index fb87d3102c..b982de34dc 100644 --- a/frame/3/herk/bli_herk_blk_var1f.c +++ b/frame/3/herk/bli_herk_blk_var1f.c @@ -52,7 +52,7 @@ void bli_herk_blk_var1f( obj_t* a, dim_t b_alg; dim_t m_trans; - if( thread_am_ochief( thread ) ) { + if( thread_am_ochief( thread ) ) { // Initialize object for packing A'. bli_obj_init_pack( &ah_pack_s ); bli_packm_init( ah, &ah_pack_s, @@ -61,9 +61,9 @@ void bli_herk_blk_var1f( obj_t* a, // Scale C by beta (if instructed). // Since scalm doesn't support multithreading yet, must be done by chief thread (ew) bli_scalm_int( &BLIS_ONE, - c, + c, cntl_sub_scalm( cntl ) ); - } + } ah_pack = thread_obroadcast( thread, &ah_pack_s ); // Initialize pack objects that are passed into packm_init() for A and C. @@ -82,9 +82,9 @@ void bli_herk_blk_var1f( obj_t* a, // Query dimension in partitioning direction. m_trans = bli_obj_length_after_trans( *c ); dim_t start, end; - bli_get_range_weighted( thread, 0, m_trans, - bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), - bli_obj_is_upper( *c ), &start, &end ); + bli_get_range_weighted_t2b( thread, 0, m_trans, + bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), + bli_obj_root_uplo( *c ), &start, &end ); // Partition along the m dimension. for ( i = start; i < end; i += b_alg ) diff --git a/frame/3/herk/bli_herk_blk_var2f.c b/frame/3/herk/bli_herk_blk_var2f.c index 2b5f3f35f7..b89842c408 100644 --- a/frame/3/herk/bli_herk_blk_var2f.c +++ b/frame/3/herk/bli_herk_blk_var2f.c @@ -90,9 +90,9 @@ void bli_herk_blk_var2f( obj_t* a, dim_t start, end; // Needs to be replaced with a weighted range because triangle - bli_get_range_weighted( thread, 0, n_trans, - bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), - bli_obj_is_lower( *c ), &start, &end ); + bli_get_range_weighted_l2r( thread, 0, n_trans, + bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), + bli_obj_root_uplo( *c ), &start, &end ); // Partition along the n dimension. for ( i = start; i < end; i += b_alg ) diff --git a/frame/3/trmm/bli_trmm_blk_var1f.c b/frame/3/trmm/bli_trmm_blk_var1f.c index eda24e3fa2..8095e0d523 100644 --- a/frame/3/trmm/bli_trmm_blk_var1f.c +++ b/frame/3/trmm/bli_trmm_blk_var1f.c @@ -94,9 +94,9 @@ void bli_trmm_blk_var1f( obj_t* a, bli_obj_width_after_trans( *a ); dim_t start, end; - bli_get_range_weighted( thread, offA, m_trans, - bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), - bli_obj_is_upper( *c ), &start, &end ); + bli_get_range_weighted_t2b( thread, offA, m_trans, + bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), + bli_obj_root_uplo( *a ), &start, &end ); // Partition along the m dimension. for ( i = start; i < end; i += b_alg ) diff --git a/frame/3/trmm/bli_trmm_blk_var2b.c b/frame/3/trmm/bli_trmm_blk_var2b.c index 6758e1a326..33b79434bf 100644 --- a/frame/3/trmm/bli_trmm_blk_var2b.c +++ b/frame/3/trmm/bli_trmm_blk_var2b.c @@ -82,9 +82,9 @@ void bli_trmm_blk_var2b( obj_t* a, // Query dimension in partitioning direction. n_trans = bli_obj_width_after_trans( *b ); dim_t start, end; - bli_get_range_weighted( thread, 0, n_trans, - bli_blksz_get_mult_for_obj( b, cntl_blocksize( cntl ) ), - bli_obj_is_upper( *c ), &start, &end ); + bli_get_range_weighted_r2l( thread, 0, n_trans, + bli_blksz_get_mult_for_obj( b, cntl_blocksize( cntl ) ), + bli_obj_root_uplo( *b ), &start, &end ); // Partition along the n dimension. for ( i = start; i < end; i += b_alg ) diff --git a/frame/3/trmm/bli_trmm_blk_var2f.c b/frame/3/trmm/bli_trmm_blk_var2f.c index 3b6d780dfc..92f1ec0d2b 100644 --- a/frame/3/trmm/bli_trmm_blk_var2f.c +++ b/frame/3/trmm/bli_trmm_blk_var2f.c @@ -82,9 +82,9 @@ void bli_trmm_blk_var2f( obj_t* a, // Query dimension in partitioning direction. n_trans = bli_obj_width_after_trans( *b ); dim_t start, end; - bli_get_range_weighted( thread, 0, n_trans, - bli_blksz_get_mult_for_obj( b, cntl_blocksize( cntl ) ), - bli_obj_is_lower( *c ), &start, &end ); + bli_get_range_weighted_l2r( thread, 0, n_trans, + bli_blksz_get_mult_for_obj( b, cntl_blocksize( cntl ) ), + bli_obj_root_uplo( *b ), &start, &end ); // Partition along the n dimension. for ( i = start; i < end; i += b_alg ) diff --git a/frame/3/trmm3/bli_trmm3_front.c b/frame/3/trmm3/bli_trmm3_front.c index f48d1481a0..9cb3f9091d 100644 --- a/frame/3/trmm3/bli_trmm3_front.c +++ b/frame/3/trmm3/bli_trmm3_front.c @@ -129,8 +129,9 @@ void bli_trmm3_front( side_t side, bli_obj_set_as_root( b_local ); bli_obj_set_as_root( c_local ); - - trmm_thrinfo_t** infos = bli_create_trmm_thrinfo_paths( FALSE ); + // Notice that, unlike trmm_r, there is no dependency in the jc loop + // for trmm3_r, so we can pass in FALSE for jc_dependency. + trmm_thrinfo_t** infos = bli_create_trmm_thrinfo_paths( FALSE ); dim_t n_threads = thread_num_threads( infos[0] ); // Invoke the internal back-end. diff --git a/frame/3/trsm/bli_trsm_blk_var1b.c b/frame/3/trsm/bli_trsm_blk_var1b.c index ca79804a1d..a79a3033a0 100644 --- a/frame/3/trsm/bli_trsm_blk_var1b.c +++ b/frame/3/trsm/bli_trsm_blk_var1b.c @@ -83,10 +83,10 @@ void bli_trsm_blk_var1b( obj_t* a, dim_t start, end; num_t dt = bli_obj_execution_datatype( *a ); - bli_get_range( thread, offA, m_trans, - //bli_lcm( bli_info_get_default_nr( BLIS_TRSM, dt ), bli_info_get_default_mr( BLIS_TRSM, dt ) ), - bli_info_get_default_mc( BLIS_TRSM, dt ), - &start, &end ); + bli_get_range_b2t( thread, offA, m_trans, + //bli_lcm( bli_info_get_default_nr( BLIS_TRSM, dt ), bli_info_get_default_mr( BLIS_TRSM, dt ) ), + bli_info_get_default_mc( BLIS_TRSM, dt ), + &start, &end ); // Partition along the remaining portion of the m dimension. for ( i = start; i < end; i += b_alg ) diff --git a/frame/3/trsm/bli_trsm_blk_var1f.c b/frame/3/trsm/bli_trsm_blk_var1f.c index fe4ecb3104..b246263329 100644 --- a/frame/3/trsm/bli_trsm_blk_var1f.c +++ b/frame/3/trsm/bli_trsm_blk_var1f.c @@ -82,10 +82,10 @@ void bli_trsm_blk_var1f( obj_t* a, dim_t start, end; num_t dt = bli_obj_execution_datatype( *a ); - bli_get_range( thread, offA, m_trans, - //bli_lcm( bli_info_get_default_nr( BLIS_TRSM, dt ), bli_info_get_default_mr( BLIS_TRSM, dt ) ), - bli_info_get_default_mc( BLIS_TRSM, dt ), - &start, &end ); + bli_get_range_t2b( thread, offA, m_trans, + //bli_lcm( bli_info_get_default_nr( BLIS_TRSM, dt ), bli_info_get_default_mr( BLIS_TRSM, dt ) ), + bli_info_get_default_mc( BLIS_TRSM, dt ), + &start, &end ); // Partition along the remaining portion of the m dimension. for ( i = start; i < end; i += b_alg ) diff --git a/frame/3/trsm/bli_trsm_blk_var2b.c b/frame/3/trsm/bli_trsm_blk_var2b.c index 82b2ea4c86..b53c5884f0 100644 --- a/frame/3/trsm/bli_trsm_blk_var2b.c +++ b/frame/3/trsm/bli_trsm_blk_var2b.c @@ -84,12 +84,12 @@ void bli_trsm_blk_var2b( obj_t* a, n_trans = bli_obj_width_after_trans( *b ); dim_t start, end; num_t dt = bli_obj_execution_datatype( *a ); - bli_get_range( thread, 0, n_trans, - //bli_lcm( bli_info_get_default_nr( BLIS_TRSM, dt ), - // bli_info_get_default_mr( BLIS_TRSM, dt ) ), - bli_lcm( bli_blksz_get_nr( dt, cntl_blocksize( cntl ) ), - bli_blksz_get_mr( dt, cntl_blocksize( cntl ) ) ), - &start, &end ); + bli_get_range_r2l( thread, 0, n_trans, + //bli_lcm( bli_info_get_default_nr( BLIS_TRSM, dt ), + // bli_info_get_default_mr( BLIS_TRSM, dt ) ), + bli_lcm( bli_blksz_get_nr( dt, cntl_blocksize( cntl ) ), + bli_blksz_get_mr( dt, cntl_blocksize( cntl ) ) ), + &start, &end ); // Partition along the n dimension. for ( i = start; i < end; i += b_alg ) diff --git a/frame/3/trsm/bli_trsm_blk_var2f.c b/frame/3/trsm/bli_trsm_blk_var2f.c index 05833d2da8..a770ec59db 100644 --- a/frame/3/trsm/bli_trsm_blk_var2f.c +++ b/frame/3/trsm/bli_trsm_blk_var2f.c @@ -84,12 +84,12 @@ void bli_trsm_blk_var2f( obj_t* a, n_trans = bli_obj_width_after_trans( *b ); dim_t start, end; num_t dt = bli_obj_execution_datatype( *a ); - bli_get_range( thread, 0, n_trans, - //bli_lcm( bli_info_get_default_nr( BLIS_TRSM, dt ), - // bli_info_get_default_mr( BLIS_TRSM, dt ) ), - bli_lcm( bli_blksz_get_nr( dt, cntl_blocksize( cntl ) ), - bli_blksz_get_mr( dt, cntl_blocksize( cntl ) ) ), - &start, &end ); + bli_get_range_l2r( thread, 0, n_trans, + //bli_lcm( bli_info_get_default_nr( BLIS_TRSM, dt ), + // bli_info_get_default_mr( BLIS_TRSM, dt ) ), + bli_lcm( bli_blksz_get_nr( dt, cntl_blocksize( cntl ) ), + bli_blksz_get_mr( dt, cntl_blocksize( cntl ) ) ), + &start, &end ); // Partition along the n dimension. for ( i = start; i < end; i += b_alg ) diff --git a/frame/3/trsm/bli_trsm_blk_var3b.c b/frame/3/trsm/bli_trsm_blk_var3b.c index bb4c2eecef..63b89a4846 100644 --- a/frame/3/trsm/bli_trsm_blk_var3b.c +++ b/frame/3/trsm/bli_trsm_blk_var3b.c @@ -130,10 +130,10 @@ void bli_trsm_blk_var3b( obj_t* a, // internal alpha scalars on A/B and C are non-zero, we must ensure // that they are only used in the first iteration. thread_ibarrier( thread ); - if ( i == 0 && thread_am_ichief( thread ) ) { + if ( i == 0 && thread_am_ichief( thread ) ) { bli_obj_scalar_reset( a ); bli_obj_scalar_reset( b ); - bli_obj_scalar_reset( c_pack ); + bli_obj_scalar_reset( c_pack ); } } diff --git a/frame/3/trsm/bli_trsm_blk_var3f.c b/frame/3/trsm/bli_trsm_blk_var3f.c index e6c43387b4..afa9c52593 100644 --- a/frame/3/trsm/bli_trsm_blk_var3f.c +++ b/frame/3/trsm/bli_trsm_blk_var3f.c @@ -130,10 +130,10 @@ void bli_trsm_blk_var3f( obj_t* a, // internal alpha scalars on A/B and C are non-zero, we must ensure // that they are only used in the first iteration. thread_ibarrier( thread ); - if ( i == 0 && thread_am_ichief( thread ) ) { + if ( i == 0 && thread_am_ichief( thread ) ) { bli_obj_scalar_reset( a ); bli_obj_scalar_reset( b ); - bli_obj_scalar_reset( c_pack ); + bli_obj_scalar_reset( c_pack ); } } diff --git a/frame/base/bli_threading.c b/frame/base/bli_threading.c index f6f0a241ce..b0a8823251 100644 --- a/frame/base/bli_threading.c +++ b/frame/base/bli_threading.c @@ -142,65 +142,319 @@ void* bli_broadcast_structure( thread_comm_t* communicator, dim_t id, void* to_s } // Code for work assignments -void bli_get_range( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, dim_t* start, dim_t* end ) -{ - thrinfo_t* thread = (thrinfo_t*) thr; - dim_t n_way = thread->n_way; - dim_t work_id = thread->work_id; - - dim_t size = all_end - all_start; - dim_t n_pt = size / n_way; - n_pt = (n_pt * n_way < size) ? n_pt + 1 : n_pt; - n_pt = (n_pt % block_factor == 0) ? n_pt : n_pt + block_factor - (n_pt % block_factor); - *start = work_id * n_pt + all_start; - *end = bli_min( *start + n_pt, size + all_start ); -} - -void bli_get_range_weighted( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, bool_t forward, dim_t* start, dim_t* end) -{ - thrinfo_t* thread = (thrinfo_t*) thr; - dim_t n_way = thread->n_way; - dim_t work_id = thread->work_id; - dim_t size = all_end - all_start; - - *start = 0; - *end = all_end - all_start; - double num = size*size / (double) n_way; - - if( forward ) { - dim_t curr_caucus = n_way - 1; - dim_t len = 0; - while(1){ - dim_t width = ceil(sqrt( len*len + num )) - len; // The width of the current caucus - width = (width % block_factor == 0) ? width : width + block_factor - (width % block_factor); - if( curr_caucus == work_id ) { - *start = bli_max( 0 , *end - width ) + all_start; - *end = *end + all_start; - return; - } - else{ - *end -= width; - len += width; - curr_caucus--; - } - } - } - else{ - while(1){ - dim_t width = ceil(sqrt(*start * *start + num)) - *start; - width = (width % block_factor == 0) ? width : width + block_factor - (width % block_factor); - - if( work_id == 0 ) { - *start = *start + all_start; - *end = bli_min( *start + width, all_end ); - return; - } - else{ - *start = *start + width; - } - work_id--; - } - } +void bli_get_range( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, bool_t handle_edge_low, dim_t* start, dim_t* end ) +{ + thrinfo_t* thread = ( thrinfo_t* )thr; + dim_t n_way = thread->n_way; + dim_t work_id = thread->work_id; + + dim_t size = all_end - all_start; + + dim_t n_bf_whole = size / block_factor; + dim_t n_bf_left = size % block_factor; + + dim_t n_bf_lo = n_bf_whole / n_way; + dim_t n_bf_hi = n_bf_whole / n_way; + + // In this function, we partition the space between all_start and + // all_end into n_way partitions, each a multiple of block_factor + // with the exception of the one partition that recieves the + // "edge" case (if applicable). + // + // Here are examples of various thread partitionings, in units of + // the block_factor, when n_way = 4. (A '+' indicates the thread + // that receives the leftover edge case (ie: n_bf_left extra + // rows/columns in its sub-range). + // (all_start ... all_end) + // n_bf_whole _left hel n_th_lo _hi thr0 thr1 thr2 thr3 + // 12 =0 f 0 4 3 3 3 3 + // 12 >0 f 0 4 3 3 3 3+ + // 13 >0 f 1 3 4 3 3 3+ + // 14 >0 f 2 2 4 4 3 3+ + // 15 >0 f 3 1 4 4 4 3+ + // 15 =0 f 3 1 4 4 4 3 + // + // 12 =0 t 4 0 3 3 3 3 + // 12 >0 t 4 0 3+ 3 3 3 + // 13 >0 t 3 1 3+ 3 3 4 + // 14 >0 t 2 2 3+ 3 4 4 + // 15 >0 t 1 3 3+ 4 4 4 + // 15 =0 t 1 3 3 4 4 4 + + // As indicated by the table above, load is balanced as equally + // as possible, even in the presence of an edge case. + + // First, we must differentiate between cases where the leftover + // "edge" case (n_bf_left) should be allocated to a thread partition + // at the low end of the index range or the high end. + + if ( handle_edge_low == FALSE ) + { + // Notice that if all threads receive the same number of + // block_factors, those threads are considered "high" and + // the "low" thread group is empty. + dim_t n_th_lo = n_bf_whole % n_way; + //dim_t n_th_hi = n_way - n_th_lo; + + // If some partitions must have more block_factors than others + // assign the slightly larger partitions to lower index threads. + if ( n_th_lo != 0 ) n_bf_lo += 1; + + // Compute the actual widths (in units of rows/columns) of + // individual threads in the low and high groups. + dim_t size_lo = n_bf_lo * block_factor; + dim_t size_hi = n_bf_hi * block_factor; + + // Precompute the starting indices of the low and high groups. + dim_t lo_start = all_start; + dim_t hi_start = all_start + n_th_lo * size_lo; + + // Compute the start and end of individual threads' ranges + // as a function of their work_ids and also the group to which + // they belong (low or high). + if ( work_id < n_th_lo ) + { + *start = lo_start + (work_id ) * size_lo; + *end = lo_start + (work_id+1) * size_lo; + } + else // if ( n_th_lo <= work_id ) + { + *start = hi_start + (work_id-n_th_lo ) * size_hi; + *end = hi_start + (work_id-n_th_lo+1) * size_hi; + + // Since the edge case is being allocated to the high + // end of the index range, we have to advance the last + // thread's end. + if ( work_id == n_way - 1 ) *end += n_bf_left; + } + } + else // if ( handle_edge_low == TRUE ) + { + // Notice that if all threads receive the same number of + // block_factors, those threads are considered "low" and + // the "high" thread group is empty. + dim_t n_th_hi = n_bf_whole % n_way; + dim_t n_th_lo = n_way - n_th_hi; + + // If some partitions must have more block_factors than others + // assign the slightly larger partitions to higher index threads. + if ( n_th_hi != 0 ) n_bf_hi += 1; + + // Compute the actual widths (in units of rows/columns) of + // individual threads in the low and high groups. + dim_t size_lo = n_bf_lo * block_factor; + dim_t size_hi = n_bf_hi * block_factor; + + // Precompute the starting indices of the low and high groups. + dim_t lo_start = all_start; + dim_t hi_start = all_start + n_th_lo * size_lo + + n_bf_left; + + // Compute the start and end of individual threads' ranges + // as a function of their work_ids and also the group to which + // they belong (low or high). + if ( work_id < n_th_lo ) + { + *start = lo_start + (work_id ) * size_lo; + *end = lo_start + (work_id+1) * size_lo; + + // Since the edge case is being allocated to the low + // end of the index range, we have to advance the + // starts/ends accordingly. + if ( work_id == 0 ) *end += n_bf_left; + else { *start += n_bf_left; + *end += n_bf_left; } + } + else // if ( n_th_lo <= work_id ) + { + *start = hi_start + (work_id-n_th_lo ) * size_hi; + *end = hi_start + (work_id-n_th_lo+1) * size_hi; + } + } +} + +void bli_get_range_l2r( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, dim_t* start, dim_t* end ) +{ + bli_get_range( thr, all_start, all_end, block_factor, + FALSE, start, end ); +} + +void bli_get_range_r2l( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, dim_t* start, dim_t* end ) +{ + bli_get_range( thr, all_start, all_end, block_factor, + TRUE, start, end ); +} + +void bli_get_range_t2b( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, dim_t* start, dim_t* end ) +{ + bli_get_range( thr, all_start, all_end, block_factor, + FALSE, start, end ); +} + +void bli_get_range_b2t( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, dim_t* start, dim_t* end ) +{ + bli_get_range( thr, all_start, all_end, block_factor, + TRUE, start, end ); +} + +void bli_get_range_weighted( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, uplo_t uplo, bool_t handle_edge_low, dim_t* start, dim_t* end ) +{ + thrinfo_t* thread = ( thrinfo_t* )thr; + dim_t n_way = thread->n_way; + dim_t work_id = thread->work_id; + dim_t size = all_end - all_start; + dim_t width; + dim_t block_fac_leftover = size % block_factor; + dim_t i; + double num; + + *start = 0; + *end = all_end - all_start; + num = size * size / ( double )n_way; + + if ( bli_is_lower( uplo ) ) + { + dim_t cur_caucus = n_way - 1; + dim_t len = 0; + + // This loop computes subpartitions backwards, from the high end + // of the index range to the low end. If the low end is assumed + // to be on the left and the high end the right, this assignment + // of widths is appropriate for n dimension partitioning of a + // lower triangular matrix. + for ( i = 0; TRUE; ++i ) + { + width = ceil( sqrt( len*len + num ) ) - len; + + // If we need to allocate the edge case (assuming it exists) + // to the high thread subpartition, adjust width so that it + // contains the exact amount of leftover edge dimension so that + // all remaining subpartitions can be multiples of block_factor. + // If the edge case is to be allocated to the low subpartition, + // or if there is no edge case, it is implicitly allocated to + // the low subpartition by virtue of the fact that all other + // subpartitions already assigned will be multiples of + // block_factor. + if ( i == 0 && !handle_edge_low ) + { + if ( width % block_factor != block_fac_leftover ) + width += block_fac_leftover - ( width % block_factor ); + } + else + { + if ( width % block_factor != 0 ) + width += block_factor - ( width % block_factor ); + } + + if ( cur_caucus == work_id ) + { + *start = bli_max( 0, *end - width ) + all_start; + *end = *end + all_start; + return; + } + else + { + *end -= width; + len += width; + cur_caucus--; + } + } + } + else // if ( bli_is_upper( uplo ) ) + { + // This loop computes subpartitions forwards, from the low end + // of the index range to the high end. If the low end is assumed + // to be on the left and the high end the right, this assignment + // of widths is appropriate for n dimension partitioning of an + // upper triangular matrix. + for ( i = 0; TRUE; ++i ) + { + width = ceil( sqrt( *start * *start + num ) ) - *start; + + if ( i == 0 && handle_edge_low ) + { + if ( width % block_factor != block_fac_leftover ) + width += block_fac_leftover - ( width % block_factor ); + } + else + { + if ( width % block_factor != 0 ) + width += block_factor - ( width % block_factor ); + } + + if ( work_id == 0 ) + { + *start = *start + all_start; + *end = bli_min( *start + width, all_end ); + return; + } + else + { + *start = *start + width; + work_id--; + } + } + } +} + +void bli_get_range_weighted_l2r( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, uplo_t uplo, dim_t* start, dim_t* end ) +{ + if ( bli_is_upper_or_lower( uplo ) ) + { + bli_get_range_weighted( thr, all_start, all_end, block_factor, + uplo, FALSE, start, end ); + } + else // if dense or zeros + { + bli_get_range_l2r( thr, all_start, all_end, block_factor, + start, end ); + } +} + +void bli_get_range_weighted_r2l( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, uplo_t uplo, dim_t* start, dim_t* end ) +{ + if ( bli_is_upper_or_lower( uplo ) ) + { +//printf( "bli_get_range_weighted_r2l: is upper or lower\n" ); + bli_toggle_uplo( uplo ); + bli_get_range_weighted( thr, all_start, all_end, block_factor, + uplo, TRUE, start, end ); + } + else // if dense or zeros + { +//printf( "bli_get_range_weighted_r2l: is dense or zeros\n" ); + bli_get_range_r2l( thr, all_start, all_end, block_factor, + start, end ); + } +} + +void bli_get_range_weighted_t2b( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, uplo_t uplo, dim_t* start, dim_t* end ) +{ + if ( bli_is_upper_or_lower( uplo ) ) + { + bli_toggle_uplo( uplo ); + bli_get_range_weighted( thr, all_start, all_end, block_factor, + uplo, FALSE, start, end ); + } + else // if dense or zeros + { + bli_get_range_t2b( thr, all_start, all_end, block_factor, + start, end ); + } +} + +void bli_get_range_weighted_b2t( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, uplo_t uplo, dim_t* start, dim_t* end ) +{ + if ( bli_is_upper_or_lower( uplo ) ) + { + bli_get_range_weighted( thr, all_start, all_end, block_factor, + uplo, TRUE, start, end ); + } + else // if dense or zeros + { + bli_get_range_b2t( thr, all_start, all_end, block_factor, + start, end ); + } } diff --git a/frame/base/bli_threading.h b/frame/base/bli_threading.h index ce4b21c553..e8aaa7bb61 100644 --- a/frame/base/bli_threading.h +++ b/frame/base/bli_threading.h @@ -127,8 +127,40 @@ typedef struct thrinfo_s thrinfo_t; #define thread_obarrier( thread ) bli_barrier( thread->ocomm, thread->ocomm_id ) #define thread_ibarrier( thread ) bli_barrier( thread->icomm, thread->icomm_id ) -void bli_get_range( void* thread, dim_t all_start, dim_t all_end, dim_t block_factor, dim_t* start, dim_t* end ); -void bli_get_range_weighted( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, bool_t forward, dim_t* start, dim_t* end); +void bli_get_range( void* thr, dim_t all_start, dim_t all_end, + dim_t block_factor, + bool_t handle_edge_low, + dim_t* start, dim_t* end ); +void bli_get_range_l2r( void* thr, dim_t all_start, dim_t all_end, + dim_t block_factor, + dim_t* start, dim_t* end ); +void bli_get_range_r2l( void* thr, dim_t all_start, dim_t all_end, + dim_t block_factor, + dim_t* start, dim_t* end ); +void bli_get_range_t2b( void* thr, dim_t all_start, dim_t all_end, + dim_t block_factor, + dim_t* start, dim_t* end ); +void bli_get_range_b2t( void* thr, dim_t all_start, dim_t all_end, + dim_t block_factor, + dim_t* start, dim_t* end ); + +void bli_get_range_weighted( void* thr, dim_t all_start, dim_t all_end, + dim_t block_factor, uplo_t uplo, + bool_t handle_edge_low, + dim_t* start, dim_t* end ); +void bli_get_range_weighted_l2r( void* thr, dim_t all_start, dim_t all_end, + dim_t block_factor, uplo_t uplo, + dim_t* start, dim_t* end ); +void bli_get_range_weighted_r2l( void* thr, dim_t all_start, dim_t all_end, + dim_t block_factor, uplo_t uplo, + dim_t* start, dim_t* end ); +void bli_get_range_weighted_t2b( void* thr, dim_t all_start, dim_t all_end, + dim_t block_factor, uplo_t uplo, + dim_t* start, dim_t* end ); +void bli_get_range_weighted_b2t( void* thr, dim_t all_start, dim_t all_end, + dim_t block_factor, uplo_t uplo, + dim_t* start, dim_t* end ); + thrinfo_t* bli_create_thread_info( thread_comm_t* ocomm, dim_t ocomm_id, thread_comm_t* icomm, dim_t icomm_id, dim_t n_way, dim_t work_id ); diff --git a/frame/include/bli_obj_macro_defs.h b/frame/include/bli_obj_macro_defs.h index 1de889ed5d..10bb01f5c2 100644 --- a/frame/include/bli_obj_macro_defs.h +++ b/frame/include/bli_obj_macro_defs.h @@ -378,12 +378,12 @@ } \ } -#define bli_obj_apply_trans( trans, obj )\ +#define bli_obj_apply_trans( trans, obj ) \ { \ (obj).info = ( (obj).info ^ (trans) ); \ } -#define bli_obj_apply_conj( conjval, obj )\ +#define bli_obj_apply_conj( conjval, obj ) \ { \ (obj).info = ( (obj).info ^ (conjval) ); \ } @@ -395,21 +395,25 @@ \ ((obj).root) +#define bli_obj_root_uplo( obj ) \ +\ + bli_obj_uplo( *bli_obj_root( obj ) ) + #define bli_obj_root_is_general( obj ) \ \ - bli_obj_is_general( *bli_obj_root( obj ) ) \ + bli_obj_is_general( *bli_obj_root( obj ) ) #define bli_obj_root_is_hermitian( obj ) \ \ - bli_obj_is_hermitian( *bli_obj_root( obj ) ) \ + bli_obj_is_hermitian( *bli_obj_root( obj ) ) #define bli_obj_root_is_symmetric( obj ) \ \ - bli_obj_is_symmetric( *bli_obj_root( obj ) ) \ + bli_obj_is_symmetric( *bli_obj_root( obj ) ) #define bli_obj_root_is_triangular( obj ) \ \ - bli_obj_is_triangular( *bli_obj_root( obj ) ) \ + bli_obj_is_triangular( *bli_obj_root( obj ) ) #define bli_obj_root_is_herm_or_symm( obj ) \ \ @@ -418,11 +422,11 @@ #define bli_obj_root_is_upper( obj ) \ \ - bli_obj_is_upper( *bli_obj_root( obj ) ) \ + bli_obj_is_upper( *bli_obj_root( obj ) ) #define bli_obj_root_is_lower( obj ) \ \ - bli_obj_is_lower( *bli_obj_root( obj ) ) \ + bli_obj_is_lower( *bli_obj_root( obj ) ) // Root matrix modification From 5f93cbe870f3478870e15581e7fd450dad5bba1e Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Thu, 11 Jun 2015 18:52:12 -0500 Subject: [PATCH 07/38] Introduced API-level initialization. Details: - Added API-level initialization state to _const, _error, _mem, _thread, _ind, and _cntl APIs. While this functionality will mostly go unused, adding miniscule overhead at init-time, there will be at least once instance in the near future where, in order to avoid an infinite loop, a certain portion of the initialization will call a query function that itself attempts to call bli_init(). API-level initialization will allow this later stage to verify that an earlier stage of initialization has completed, even if the overall call to bli_init() has not yet returned. - Added _is_initialized() functions for each API, setting the underlying bool_t during _init() and unsetting it during _finalize(). - Comment, whitespace changes. --- frame/base/bli_const.c | 16 ++++- frame/base/bli_const.h | 5 +- frame/base/bli_error.c | 103 +++++++++++++++++++------------- frame/base/bli_error.h | 13 ++-- frame/base/bli_init.c | 35 ++++++----- frame/base/bli_init.h | 9 +-- frame/base/bli_mem.c | 16 ++++- frame/base/bli_mem.h | 8 ++- frame/base/bli_threading.c | 15 ++++- frame/base/bli_threading.h | 1 + frame/cntl/bli_cntl_init.c | 13 ++++ frame/cntl/bli_cntl_init.h | 5 +- frame/ind/query/bli_bsv_query.c | 3 +- frame/ind/query/bli_ind_query.c | 12 +++- frame/ind/query/bli_ind_query.h | 1 + frame/ind/query/bli_ukr_query.c | 6 +- 16 files changed, 178 insertions(+), 83 deletions(-) diff --git a/frame/base/bli_const.c b/frame/base/bli_const.c index 316f115a45..7eaebf5e28 100644 --- a/frame/base/bli_const.c +++ b/frame/base/bli_const.c @@ -34,9 +34,6 @@ #include "blis.h" -// These globally-declared obj_t's are used to store the so-called -// "global scalar constants" for commonly-used 1x1 values. - obj_t BLIS_TWO; obj_t BLIS_ONE; obj_t BLIS_ONE_HALF; @@ -45,6 +42,8 @@ obj_t BLIS_MINUS_ONE_HALF; obj_t BLIS_MINUS_ONE; obj_t BLIS_MINUS_TWO; +static bool_t bli_const_is_init = FALSE; + void bli_const_init( void ) { bli_obj_create_const( 2.0, &BLIS_TWO ); @@ -54,6 +53,9 @@ void bli_const_init( void ) bli_obj_create_const( -0.5, &BLIS_MINUS_ONE_HALF ); bli_obj_create_const( -1.0, &BLIS_MINUS_ONE ); bli_obj_create_const( -2.0, &BLIS_MINUS_TWO ); + + // Mark API as initialized. + bli_const_is_init = TRUE; } void bli_const_finalize( void ) @@ -65,5 +67,13 @@ void bli_const_finalize( void ) bli_obj_free( &BLIS_MINUS_ONE_HALF ); bli_obj_free( &BLIS_MINUS_ONE ); bli_obj_free( &BLIS_MINUS_TWO ); + + // Mark API as uninitialized. + bli_const_is_init = FALSE; +} + +bool_t bli_const_is_initialized( void ) +{ + return bli_const_is_init; } diff --git a/frame/base/bli_const.h b/frame/base/bli_const.h index 066fd5e72a..73cfafd10e 100644 --- a/frame/base/bli_const.h +++ b/frame/base/bli_const.h @@ -32,6 +32,7 @@ */ -void bli_const_init( void ); -void bli_const_finalize( void ); +void bli_const_init( void ); +void bli_const_finalize( void ); +bool_t bli_const_is_initialized( void ); diff --git a/frame/base/bli_error.c b/frame/base/bli_error.c index 11064a72ac..b8c18f77e1 100644 --- a/frame/base/bli_error.c +++ b/frame/base/bli_error.c @@ -34,59 +34,33 @@ #include "blis.h" -// Current error checking level. -static errlev_t bli_err_chk_level = BLIS_FULL_ERROR_CHECKING; - -// Internal array to hold error strings. -static char bli_error_string[BLIS_MAX_NUM_ERR_MSGS][BLIS_MAX_ERR_MSG_LENGTH]; - +static bool_t bli_error_is_init = FALSE; -errlev_t bli_error_checking_level() -{ - return bli_err_chk_level; -} - -errlev_t bli_error_checking_level_set( errlev_t new_level ) +void bli_error_init( void ) { - err_t e_val; - errlev_t old_level; - - e_val = bli_check_valid_error_level( new_level ); - bli_check_error_code( e_val ); + bli_error_init_msgs(); - old_level = bli_err_chk_level; - - bli_err_chk_level = new_level; - - return old_level; + // Mark API as initialized. + bli_error_is_init = TRUE; } -bool_t bli_error_checking_is_enabled() +void bli_error_finalize( void ) { - return bli_error_checking_level() != BLIS_NO_ERROR_CHECKING; + // Mark API as uninitialized. + bli_error_is_init = FALSE; } -char* bli_error_string_for_code( gint_t code ) +bool_t bli_error_is_initialized( void ) { - return bli_error_string[-code]; + return bli_error_is_init; } -void bli_abort( void ) -{ - fprintf( stderr, "libblis: Aborting.\n" ); - //raise( SIGABRT ); - abort(); -} +// ----------------------------------------------------------------------------- -void bli_print_msg( char* str, char* file, guint_t line ) -{ - fprintf( stderr, "\n" ); - fprintf( stderr, "libblis: %s (line %lu):\n", file, ( long unsigned int )line ); - fprintf( stderr, "libblis: %s\n", str ); - fflush( stderr ); -} +// Internal array to hold error strings. +static char bli_error_string[BLIS_MAX_NUM_ERR_MSGS][BLIS_MAX_ERR_MSG_LENGTH]; -void bli_error_init( void ) +void bli_error_init_msgs( void ) { sprintf( bli_error_string_for_code(BLIS_INVALID_ERROR_CHECKING_LEVEL), "Invalid error checking level." ); @@ -196,8 +170,53 @@ void bli_error_init( void ) "Expected object to be alias." ); } -void bli_error_finalize( void ) +void bli_print_msg( char* str, char* file, guint_t line ) { - // Nothing to do. + fprintf( stderr, "\n" ); + fprintf( stderr, "libblis: %s (line %lu):\n", file, ( long unsigned int )line ); + fprintf( stderr, "libblis: %s\n", str ); + fflush( stderr ); +} + +void bli_abort( void ) +{ + fprintf( stderr, "libblis: Aborting.\n" ); + //raise( SIGABRT ); + abort(); +} + +// ----------------------------------------------------------------------------- + +// Current error checking level. +static errlev_t bli_err_chk_level = BLIS_FULL_ERROR_CHECKING; + +errlev_t bli_error_checking_level( void ) +{ + return bli_err_chk_level; +} + +errlev_t bli_error_checking_level_set( errlev_t new_level ) +{ + err_t e_val; + errlev_t old_level; + + e_val = bli_check_valid_error_level( new_level ); + bli_check_error_code( e_val ); + + old_level = bli_err_chk_level; + + bli_err_chk_level = new_level; + + return old_level; +} + +bool_t bli_error_checking_is_enabled( void ) +{ + return bli_error_checking_level() != BLIS_NO_ERROR_CHECKING; +} + +char* bli_error_string_for_code( gint_t code ) +{ + return bli_error_string[-code]; } diff --git a/frame/base/bli_error.h b/frame/base/bli_error.h index 6f16472b08..1dcc9b0b2a 100644 --- a/frame/base/bli_error.h +++ b/frame/base/bli_error.h @@ -33,6 +33,14 @@ */ +void bli_error_init( void ); +void bli_error_finalize( void ); +bool_t bli_error_is_initialized( void ); + +void bli_error_init_msgs( void ); +void bli_print_msg( char* str, char* file, guint_t line ); +void bli_abort( void ); + errlev_t bli_error_checking_level( void ); errlev_t bli_error_checking_level_set( errlev_t new_level ); @@ -40,9 +48,4 @@ bool_t bli_error_checking_is_enabled( void ); char* bli_error_string_for_code( gint_t code ); -void bli_abort( void ); -void bli_print_msg( char* str, char* file, guint_t line ); - -void bli_error_init( void ); -void bli_error_finalize( void ); diff --git a/frame/base/bli_init.c b/frame/base/bli_init.c index b69e8086d0..7a0c0f39b1 100644 --- a/frame/base/bli_init.c +++ b/frame/base/bli_init.c @@ -38,22 +38,22 @@ pthread_mutex_t initialize_mutex = PTHREAD_MUTEX_INITIALIZER; #endif -static bool_t bli_initialized = FALSE; +static bool_t bli_is_init = FALSE; err_t bli_init( void ) { err_t r_val = BLIS_FAILURE; - // If bli_initialized is TRUE, then we know without a doubt that + // If bli_is_init is TRUE, then we know without a doubt that // BLIS is presently initialized, and thus we can return early. - if ( bli_initialized == TRUE ) return r_val; + if ( bli_is_init == TRUE ) return r_val; - // NOTE: if bli_initialized is FALSE, we cannot be certain that BLIS + // NOTE: if bli_is_init is FALSE, we cannot be certain that BLIS // is ready to be initialized; it may be the case that a thread is // inside the critical section below and is already in the process // of initializing BLIS, but has not yet finished and updated - // bli_initialized accordingly. This boolean asymmetry is important! + // bli_is_init accordingly. This boolean asymmetry is important! // We enclose the bodies of bli_init() and bli_finalize() in a // critical section (both with the same name) so that they can be @@ -75,10 +75,10 @@ err_t bli_init( void ) // Proceed with initialization only if BLIS is presently uninitialized. // Since we bli_init() and bli_finalize() use the same named critical // section, we can be sure that no other thread is either (a) updating - // bli_initialized, or (b) testing bli_initialized within the critical + // bli_is_init, or (b) testing bli_is_init within the critical // section (for the purposes of deciding whether to perform the // necessary initialization subtasks). - if ( bli_initialized == FALSE ) + if ( bli_is_init == FALSE ) { // Initialize various sub-APIs. bli_const_init(); @@ -89,7 +89,7 @@ err_t bli_init( void ) bli_thread_init(); // After initialization is complete, mark BLIS as initialized. - bli_initialized = TRUE; + bli_is_init = TRUE; // Only the thread that actually performs the initialization will // return "success". @@ -109,15 +109,15 @@ err_t bli_finalize( void ) { err_t r_val = BLIS_FAILURE; - // If bli_initialized is FALSE, then we know without a doubt that + // If bli_is_init is FALSE, then we know without a doubt that // BLIS is presently uninitialized, and thus we can return early. - if ( bli_initialized == FALSE ) return r_val; + if ( bli_is_init == FALSE ) return r_val; - // NOTE: if bli_initialized is TRUE, we cannot be certain that BLIS + // NOTE: if bli_is_init is TRUE, we cannot be certain that BLIS // is ready to be finalized; it may be the case that a thread is // inside the critical section below and is already in the process // of finalizing BLIS, but has not yet finished and updated - // bli_initialized accordingly. This boolean asymmetry is important! + // bli_is_init accordingly. This boolean asymmetry is important! // We enclose the bodies of bli_init() and bli_finalize() in a // critical section (both with the same name) so that they can be @@ -139,10 +139,10 @@ err_t bli_finalize( void ) // Proceed with finalization only if BLIS is presently initialized. // Since we bli_init() and bli_finalize() use the same named critical // section, we can be sure that no other thread is either (a) updating - // bli_initialized, or (b) testing bli_initialized within the critical + // bli_is_init, or (b) testing bli_is_init within the critical // section (for the purposes of deciding whether to perform the // necessary finalization subtasks). - if ( bli_initialized == TRUE ) + if ( bli_is_init == TRUE ) { // Finalize various sub-APIs. bli_const_finalize(); @@ -153,7 +153,7 @@ err_t bli_finalize( void ) bli_thread_finalize(); // After finalization is complete, mark BLIS as uninitialized. - bli_initialized = FALSE; + bli_is_init = FALSE; // Only the thread that actually performs the finalization will // return "success". @@ -173,6 +173,11 @@ err_t bli_finalize( void ) return r_val; } +bool_t bli_is_initialized( void ) +{ + return bli_is_init; +} + void bli_init_auto( err_t* init_result ) { *init_result = bli_init(); diff --git a/frame/base/bli_init.h b/frame/base/bli_init.h index cd620008df..d1e9cc0860 100644 --- a/frame/base/bli_init.h +++ b/frame/base/bli_init.h @@ -32,8 +32,9 @@ */ -err_t bli_init( void ); -err_t bli_finalize( void ); +err_t bli_init( void ); +err_t bli_finalize( void ); +bool_t bli_is_initialized( void ); -void bli_init_auto( err_t* init_result ); -void bli_finalize_auto( err_t init_result ); +void bli_init_auto( err_t* init_result ); +void bli_finalize_auto( err_t init_result ); diff --git a/frame/base/bli_mem.c b/frame/base/bli_mem.c index cd5717e310..6557518017 100644 --- a/frame/base/bli_mem.c +++ b/frame/base/bli_mem.c @@ -34,6 +34,8 @@ #include "blis.h" +static bool_t bli_mem_is_init = FALSE; + #ifdef BLIS_ENABLE_PTHREADS pthread_mutex_t mem_manager_mutex = PTHREAD_MUTEX_INITIALIZER; #endif @@ -259,7 +261,7 @@ void bli_mem_acquire_v( siz_t req_size, -void bli_mem_init() +void bli_mem_init( void ) { dim_t index_a; dim_t index_b; @@ -307,6 +309,9 @@ void bli_mem_init() #ifdef BLIS_ENABLE_PTHREADS pthread_mutex_unlock( &mem_manager_mutex ); #endif + + // Mark API as initialized. + bli_mem_is_init = TRUE; } @@ -359,7 +364,7 @@ void bli_mem_init_pool( char* pool_mem, -void bli_mem_finalize() +void bli_mem_finalize( void ) { #ifdef BLIS_ENABLE_OPENMP @@ -383,5 +388,12 @@ void bli_mem_finalize() pthread_mutex_destroy( &mem_manager_mutex ); #endif + // Mark API as uninitialized. + bli_mem_is_init = FALSE; +} + +bool_t bli_mem_is_initialized( void ) +{ + return bli_mem_is_init; } diff --git a/frame/base/bli_mem.h b/frame/base/bli_mem.h index 6e83ac1223..8fc6d85d6f 100644 --- a/frame/base/bli_mem.h +++ b/frame/base/bli_mem.h @@ -32,6 +32,11 @@ */ +void bli_mem_init( void ); +void bli_mem_finalize( void ); +bool_t bli_mem_is_initialized( void ); + + void bli_mem_acquire_m( siz_t req_size, packbuf_t buf_type, mem_t* mem ); @@ -41,9 +46,6 @@ void bli_mem_acquire_v( siz_t req_size, void bli_mem_release( mem_t* mem ); -void bli_mem_init( void ); -void bli_mem_finalize( void ); - void bli_mem_init_pool( char* pool_mem, siz_t block_size, dim_t n_blocks, diff --git a/frame/base/bli_threading.c b/frame/base/bli_threading.c index b0a8823251..f3e50f61e3 100644 --- a/frame/base/bli_threading.c +++ b/frame/base/bli_threading.c @@ -34,6 +34,8 @@ #include "blis.h" +static bool_t bli_thread_is_init = FALSE; + packm_thrinfo_t BLIS_PACKM_SINGLE_THREADED; gemm_thrinfo_t BLIS_GEMM_SINGLE_THREADED; herk_thrinfo_t BLIS_HERK_SINGLE_THREADED; @@ -46,13 +48,24 @@ void bli_thread_init( void ) bli_setup_packm_single_threaded_info( &BLIS_PACKM_SINGLE_THREADED ); bli_setup_gemm_single_threaded_info( &BLIS_GEMM_SINGLE_THREADED ); bli_setup_herk_single_threaded_info( &BLIS_HERK_SINGLE_THREADED ); + + // Mark API as initialized. + bli_thread_is_init = TRUE; } void bli_thread_finalize( void ) { - // Nothing to do. + // Mark API as uninitialized. + bli_thread_is_init = FALSE; +} + +bool_t bli_thread_is_initialized( void ) +{ + return bli_thread_is_init; } +// ----------------------------------------------------------------------------- + //*********** Stuff Specific to single-threaded ************* #ifndef BLIS_ENABLE_MULTITHREADING void bli_barrier( thread_comm_t* communicator, dim_t t_id ) diff --git a/frame/base/bli_threading.h b/frame/base/bli_threading.h index e8aaa7bb61..89b4384748 100644 --- a/frame/base/bli_threading.h +++ b/frame/base/bli_threading.h @@ -89,6 +89,7 @@ typedef struct thread_comm_s thread_comm_t; void bli_thread_init( void ); void bli_thread_finalize( void ); +bool_t bli_thread_is_initialized( void ); // Thread Communicator Interface Definitions void bli_setup_communicator( thread_comm_t* communicator, dim_t n_threads ); diff --git a/frame/cntl/bli_cntl_init.c b/frame/cntl/bli_cntl_init.c index 8c005c3786..e2443c7b52 100644 --- a/frame/cntl/bli_cntl_init.c +++ b/frame/cntl/bli_cntl_init.c @@ -34,6 +34,8 @@ #include "blis.h" +static bool_t bli_cntl_is_init = FALSE; + void bli_cntl_init( void ) { // Level-1 @@ -61,6 +63,9 @@ void bli_cntl_init( void ) // Level-3 induced bli_ind_cntl_init(); + + // Mark API as initialized. + bli_cntl_is_init = TRUE; } void bli_cntl_finalize( void ) @@ -90,5 +95,13 @@ void bli_cntl_finalize( void ) // Level-3 induced bli_ind_cntl_finalize(); + + // Mark API as uninitialized. + bli_cntl_is_init = FALSE; +} + +bool_t bli_cntl_is_initialized( void ) +{ + return bli_cntl_is_init; } diff --git a/frame/cntl/bli_cntl_init.h b/frame/cntl/bli_cntl_init.h index c1bce09e05..a3fdf6279f 100644 --- a/frame/cntl/bli_cntl_init.h +++ b/frame/cntl/bli_cntl_init.h @@ -32,5 +32,6 @@ */ -void bli_cntl_init( void ); -void bli_cntl_finalize( void ); +void bli_cntl_init( void ); +void bli_cntl_finalize( void ); +bool_t bli_cntl_is_initialized( void ); diff --git a/frame/ind/query/bli_bsv_query.c b/frame/ind/query/bli_bsv_query.c index 4c43cb8421..5124a19939 100644 --- a/frame/ind/query/bli_bsv_query.c +++ b/frame/ind/query/bli_bsv_query.c @@ -169,7 +169,8 @@ blksz_t* bli_bsv_get_blksz( bszid_t bsv, ind_t method ) { // Initialize BLIS, if it isn't already initialized. This is // needed because we have to ensure that the blksz_t objects - // have been created. + // have been created, otherwise this function could return a + // NULL (or garbage) address. bli_init(); return *(bli_bsizes[ method ][ bsv ]); diff --git a/frame/ind/query/bli_ind_query.c b/frame/ind/query/bli_ind_query.c index c48e1a6859..6ebf92aeae 100644 --- a/frame/ind/query/bli_ind_query.c +++ b/frame/ind/query/bli_ind_query.c @@ -34,6 +34,7 @@ #include "blis.h" +static bool_t bli_ind_is_init = FALSE; static void* bli_ind_oper_fp[BLIS_NUM_IND_METHODS][BLIS_NUM_LEVEL3_OPS] = { @@ -220,11 +221,20 @@ void bli_ind_init( void ) #ifdef BLIS_ENABLE_INDUCED_DCOMPLEX bli_ind_enable_dt( BLIS_4M1A, BLIS_DCOMPLEX ); #endif + + // Mark API as initialized. + bli_ind_is_init = TRUE; } void bli_ind_finalize( void ) { - // Nothing to do. + // Mark API as uninitialized. + bli_ind_is_init = FALSE; +} + +bool_t bli_ind_is_initialized( void ) +{ + return bli_ind_is_init; } // ----------------------------------------------------------------------------- diff --git a/frame/ind/query/bli_ind_query.h b/frame/ind/query/bli_ind_query.h index 4b199ff8e0..6b1437c940 100644 --- a/frame/ind/query/bli_ind_query.h +++ b/frame/ind/query/bli_ind_query.h @@ -84,6 +84,7 @@ char* bli_ind_oper_get_avail_impl_string( opid_t oper, num_t dt ); void bli_ind_init( void ); void bli_ind_finalize( void ); +bool_t bli_ind_is_initialized( void ); void bli_ind_enable( ind_t method ); void bli_ind_disable( ind_t method ); diff --git a/frame/ind/query/bli_ukr_query.c b/frame/ind/query/bli_ukr_query.c index d36a9d21ab..464016300e 100644 --- a/frame/ind/query/bli_ukr_query.c +++ b/frame/ind/query/bli_ukr_query.c @@ -205,7 +205,8 @@ func_t* bli_ukr_get_funcs( l3ukr_t ukr, ind_t method ) // have been created (and thus contain valid function pointers). bli_init(); - // Avoid dereferencing NULL pointers. + // Avoid dereferencing NULL pointers. (A NULL pointer indicates that + // there is no kernel for the requested kernel type and method.) if ( p == NULL ) return NULL; else return *p; } @@ -219,7 +220,8 @@ func_t* bli_ukr_get_ref_funcs( l3ukr_t ukr ) // have been created (and thus contain valid function pointers). bli_init(); - // Avoid dereferencing NULL pointers. + // Avoid dereferencing NULL pointers. (A NULL pointer indicates that + // there is no reference kernel for the requested kernel type.) if ( p == NULL ) return NULL; else return *p; } From 9848f255a3bab17d1139c391cca13ff3f1ffe6ed Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Thu, 11 Jun 2015 19:14:22 -0500 Subject: [PATCH 08/38] Added early return to API-level _init() routines. Details: - Added conditional code that returns early from the API-level _init() routines if the API is already initialized. Actually meant for this to be included in 5f93cbe8. --- frame/base/bli_const.c | 3 +++ frame/base/bli_error.c | 3 +++ frame/base/bli_mem.c | 3 +++ frame/base/bli_threading.c | 2 ++ frame/cntl/bli_cntl_init.c | 3 +++ frame/ind/query/bli_ind_query.c | 3 +++ 6 files changed, 17 insertions(+) diff --git a/frame/base/bli_const.c b/frame/base/bli_const.c index 7eaebf5e28..4c7d9effc4 100644 --- a/frame/base/bli_const.c +++ b/frame/base/bli_const.c @@ -46,6 +46,9 @@ static bool_t bli_const_is_init = FALSE; void bli_const_init( void ) { + // If the API is already initialized, return early. + if ( bli_const_is_initialized() ) return; + bli_obj_create_const( 2.0, &BLIS_TWO ); bli_obj_create_const( 1.0, &BLIS_ONE ); bli_obj_create_const( 0.5, &BLIS_ONE_HALF ); diff --git a/frame/base/bli_error.c b/frame/base/bli_error.c index b8c18f77e1..18c84ab659 100644 --- a/frame/base/bli_error.c +++ b/frame/base/bli_error.c @@ -38,6 +38,9 @@ static bool_t bli_error_is_init = FALSE; void bli_error_init( void ) { + // If the API is already initialized, return early. + if ( bli_error_is_initialized() ) return; + bli_error_init_msgs(); // Mark API as initialized. diff --git a/frame/base/bli_mem.c b/frame/base/bli_mem.c index 6557518017..6f13d3c1a6 100644 --- a/frame/base/bli_mem.c +++ b/frame/base/bli_mem.c @@ -263,6 +263,9 @@ void bli_mem_acquire_v( siz_t req_size, void bli_mem_init( void ) { + // If the API is already initialized, return early. + if ( bli_mem_is_initialized() ) return; + dim_t index_a; dim_t index_b; dim_t index_c; diff --git a/frame/base/bli_threading.c b/frame/base/bli_threading.c index f3e50f61e3..890a1d391e 100644 --- a/frame/base/bli_threading.c +++ b/frame/base/bli_threading.c @@ -43,6 +43,8 @@ thread_comm_t BLIS_SINGLE_COMM; void bli_thread_init( void ) { + // If the API is already initialized, return early. + if ( bli_thread_is_initialized() ) return; bli_setup_communicator( &BLIS_SINGLE_COMM, 1 ); bli_setup_packm_single_threaded_info( &BLIS_PACKM_SINGLE_THREADED ); diff --git a/frame/cntl/bli_cntl_init.c b/frame/cntl/bli_cntl_init.c index e2443c7b52..54187d9094 100644 --- a/frame/cntl/bli_cntl_init.c +++ b/frame/cntl/bli_cntl_init.c @@ -38,6 +38,9 @@ static bool_t bli_cntl_is_init = FALSE; void bli_cntl_init( void ) { + // If the API is already initialized, return early. + if ( bli_cntl_is_initialized() ) return; + // Level-1 bli_scalv_cntl_init(); bli_packv_cntl_init(); diff --git a/frame/ind/query/bli_ind_query.c b/frame/ind/query/bli_ind_query.c index 6ebf92aeae..1e0f563978 100644 --- a/frame/ind/query/bli_ind_query.c +++ b/frame/ind/query/bli_ind_query.c @@ -215,6 +215,9 @@ char* bli_ind_oper_get_avail_impl_string( opid_t oper, num_t dt ) void bli_ind_init( void ) { + // If the API is already initialized, return early. + if ( bli_ind_is_initialized() ) return; + #ifdef BLIS_ENABLE_INDUCED_SCOMPLEX bli_ind_enable_dt( BLIS_4M1A, BLIS_SCOMPLEX ); #endif From 7cd01b71b5e757a6774625b3c9f427f5e7664a76 Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Fri, 19 Jun 2015 11:31:53 -0500 Subject: [PATCH 09/38] Implemented dynamic allocation for packing buffers. Details: - Replaced the old memory allocator, which was based on statically- allocated arrays, with one based on a new internal pool_t type, which, combined with a new bli_pool_*() API, provides a new abstract data type that implements the same memory pool functionality but with blocks from the heap (ie: malloc() or equivalent). Hiding the details of the pool in a separate API also allows for a much simpler bli_mem.c family of functions. - Added a new internal header, bli_config_macro_defs.h, which enables sane defaults for the values previously found in bli_config. Those values can be overridden by #defining them in bli_config.h the same way kernel defaults can be overridden in bli_kernel.h. This file most resembles what was previously a typical configuration's bli_config.h. - Added a new configuration macro, BLIS_POOL_ADDR_ALIGN_SIZE, which defaults to BLIS_PAGE_SIZE, to specify the alignment of individual blocks in the memory pool. Also added a corresponding query routine to the bli_info API. - Deprecated (once again) the micro-panel alignment feature. Upon further reflection, it seems that the goal of more predictable L1 cache replacement behavior is outweighed by the harm caused by non-contiguous micro-panels when k % kc != 0. I honestly don't think anyone will even miss this feature. - Changed bli_ukr_get_funcs() and bli_ukr_get_ref_funcs() to call bli_cntl_init() instead of bli_init(). - Removed query functions from bli_info.c that are no longer applicable given the dynamic memory allocator. - Removed unnecessary definitions from configurations' bli_config.h files, which are now pleasantly sparse. - Fixed incorrect flop counts in addv, subv, scal2v, scal2m testsuite modules. Thanks to Devangi Parikh for pointing out these miscalculations. - Comment, whitespace changes. --- config/armv7a/bli_config.h | 134 ----- config/armv8a/bli_config.h | 133 ----- config/bgq/bli_config.h | 136 ----- config/bulldozer/bli_config.h | 136 +---- config/cortex-a15/bli_config.h | 133 ----- config/cortex-a9/bli_config.h | 133 ----- config/dunnington/bli_config.h | 137 +---- config/emscripten/bli_config.h | 133 ----- config/loongson3a/bli_config.h | 133 ----- config/mic/bli_config.h | 136 ----- config/mic/bli_kernel.h | 2 + config/piledriver/bli_config.h | 139 +---- config/pnacl/bli_config.h | 133 ----- config/power7/bli_config.h | 133 ----- config/reference/bli_config.h | 137 ----- config/sandybridge/bli_config.h | 137 ----- config/template/bli_config.h | 138 +---- frame/1m/packm/bli_packm_init.c | 49 -- frame/3/gemm/bli_gemm_cntl.c | 21 - frame/base/bli_info.c | 57 +- frame/base/bli_info.h | 25 +- frame/base/bli_init.c | 106 ++-- frame/base/bli_mem.c | 498 ++++++++++------ frame/base/bli_mem.c.prev | 366 ++++++++++++ frame/base/bli_mem.h | 18 +- frame/base/bli_pool.c | 344 +++++++++++ .../bli_pool_macro_defs.h => base/bli_pool.h} | 117 +++- frame/include/bli_config_macro_defs.h | 178 ++++++ frame/include/bli_kernel_macro_defs.h | 35 -- frame/include/bli_macro_defs.h | 6 +- frame/include/bli_mem_macro_defs.h | 40 +- frame/include/bli_mem_pool_macro_defs.h | 535 ------------------ frame/include/bli_type_defs.h | 54 +- frame/include/blis.h | 6 +- frame/ind/query/bli_bsv_query.c | 10 +- frame/ind/query/bli_ind_query.c | 4 +- frame/ind/query/bli_ukr_query.c | 16 +- testsuite/src/test_addv.c | 2 +- testsuite/src/test_libblis.c | 54 +- testsuite/src/test_scal2m.c | 2 +- testsuite/src/test_scal2v.c | 2 +- testsuite/src/test_subv.c | 2 +- 42 files changed, 1499 insertions(+), 3211 deletions(-) create mode 100644 frame/base/bli_mem.c.prev create mode 100644 frame/base/bli_pool.c rename frame/{include/bli_pool_macro_defs.h => base/bli_pool.h} (54%) create mode 100644 frame/include/bli_config_macro_defs.h delete mode 100644 frame/include/bli_mem_pool_macro_defs.h diff --git a/config/armv7a/bli_config.h b/config/armv7a/bli_config.h index a757069e1f..21041fc2a4 100644 --- a/config/armv7a/bli_config.h +++ b/config/armv7a/bli_config.h @@ -36,142 +36,8 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. #define BLIS_INT_TYPE_SIZE 32 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 1 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 32 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. -#define BLIS_SIMD_ALIGN_SIZE 32 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. #define BLIS_BLAS2BLIS_INT_TYPE_SIZE 32 -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - - #endif diff --git a/config/armv8a/bli_config.h b/config/armv8a/bli_config.h index 72cc930e4d..127a2bca70 100644 --- a/config/armv8a/bli_config.h +++ b/config/armv8a/bli_config.h @@ -36,142 +36,9 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. #define BLIS_INT_TYPE_SIZE 64 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 1 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. #define BLIS_SIMD_ALIGN_SIZE 16 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. #define BLIS_BLAS2BLIS_INT_TYPE_SIZE 32 -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - - #endif diff --git a/config/bgq/bli_config.h b/config/bgq/bli_config.h index 09396b6a01..9810b5c116 100644 --- a/config/bgq/bli_config.h +++ b/config/bgq/bli_config.h @@ -38,144 +38,8 @@ #undef restrict - -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. -#define BLIS_INT_TYPE_SIZE 64 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 64 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS 1 -#define BLIS_NUM_KC_X_NC_BLOCKS 1 -#define BLIS_NUM_MC_X_NC_BLOCKS 1 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. -#define BLIS_SIMD_ALIGN_SIZE 32 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE 32 - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE 64 - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE 64 - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE -#define BLIS_CONTIG_STRIDE_ALIGN_SIZE 32 - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. #define BLIS_BLAS2BLIS_INT_TYPE_SIZE 32 -// Fortran-77 name-mangling macros. -// Underscore is left out to work on BGQ systems -#define PASTEF770(name) name //## _ -#define PASTEF77(ch1,name) ch1 ## name //## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name //## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - #endif diff --git a/config/bulldozer/bli_config.h b/config/bulldozer/bli_config.h index 4b220da1e5..1f99e7e538 100644 --- a/config/bulldozer/bli_config.h +++ b/config/bulldozer/bli_config.h @@ -36,141 +36,7 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. -#define BLIS_INT_TYPE_SIZE 64 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 4 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. -#define BLIS_SIMD_ALIGN_SIZE 16 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. -#define BLIS_BLAS2BLIS_INT_TYPE_SIZE 64 - -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - +#define BLIS_SIMD_ALIGN_SIZE 16 diff --git a/config/cortex-a15/bli_config.h b/config/cortex-a15/bli_config.h index a05f8d0827..7537125400 100644 --- a/config/cortex-a15/bli_config.h +++ b/config/cortex-a15/bli_config.h @@ -36,142 +36,9 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. #define BLIS_INT_TYPE_SIZE 32 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 1 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. #define BLIS_SIMD_ALIGN_SIZE 16 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. #define BLIS_BLAS2BLIS_INT_TYPE_SIZE 32 -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - - #endif diff --git a/config/cortex-a9/bli_config.h b/config/cortex-a9/bli_config.h index a05f8d0827..7537125400 100644 --- a/config/cortex-a9/bli_config.h +++ b/config/cortex-a9/bli_config.h @@ -36,142 +36,9 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. #define BLIS_INT_TYPE_SIZE 32 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 1 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. #define BLIS_SIMD_ALIGN_SIZE 16 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. #define BLIS_BLAS2BLIS_INT_TYPE_SIZE 32 -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - - #endif diff --git a/config/dunnington/bli_config.h b/config/dunnington/bli_config.h index 13cb2c1f64..64392de999 100644 --- a/config/dunnington/bli_config.h +++ b/config/dunnington/bli_config.h @@ -36,142 +36,7 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. -#define BLIS_INT_TYPE_SIZE 64 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 1 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS 1 -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. -#define BLIS_SIMD_ALIGN_SIZE 16 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE 16 - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE 16 - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. -#define BLIS_BLAS2BLIS_INT_TYPE_SIZE 64 - -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - +#define BLIS_SIMD_ALIGN_SIZE 16 #endif diff --git a/config/emscripten/bli_config.h b/config/emscripten/bli_config.h index a05f8d0827..7537125400 100644 --- a/config/emscripten/bli_config.h +++ b/config/emscripten/bli_config.h @@ -36,142 +36,9 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. #define BLIS_INT_TYPE_SIZE 32 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 1 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. #define BLIS_SIMD_ALIGN_SIZE 16 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. #define BLIS_BLAS2BLIS_INT_TYPE_SIZE 32 -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - - #endif diff --git a/config/loongson3a/bli_config.h b/config/loongson3a/bli_config.h index 42ae3a4bf1..7537125400 100644 --- a/config/loongson3a/bli_config.h +++ b/config/loongson3a/bli_config.h @@ -36,142 +36,9 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. #define BLIS_INT_TYPE_SIZE 32 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 8 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS 1 -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. #define BLIS_SIMD_ALIGN_SIZE 16 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE 16 - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE 16 - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE 16 - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. #define BLIS_BLAS2BLIS_INT_TYPE_SIZE 32 -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - - #endif diff --git a/config/mic/bli_config.h b/config/mic/bli_config.h index 8d11e5da80..a119a2dde1 100644 --- a/config/mic/bli_config.h +++ b/config/mic/bli_config.h @@ -39,146 +39,10 @@ #define BLIS_TREE_BARRIER #define BLIS_TREE_BARRIER_ARITY 4 -#define BLIS_DGEMM_UKERNEL_PREFERS_CONTIG_ROWS -#define BLIS_ENABLE_MULTITHREADING #define BLIS_ENABLE_OPENMP -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. -#define BLIS_INT_TYPE_SIZE 64 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 60 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS 1 -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 256 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. #define BLIS_SIMD_ALIGN_SIZE 32 -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. -#define BLIS_BLAS2BLIS_INT_TYPE_SIZE 64 - -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - #endif diff --git a/config/mic/bli_kernel.h b/config/mic/bli_kernel.h index 8e86791223..880e97d358 100644 --- a/config/mic/bli_kernel.h +++ b/config/mic/bli_kernel.h @@ -151,6 +151,8 @@ // -- gemm -- +#define BLIS_DGEMM_UKERNEL_PREFERS_CONTIG_ROWS + #define BLIS_DGEMM_UKERNEL bli_dgemm_opt_30x8 #define BLIS_SGEMM_UKERNEL bli_sgemm_opt_30x16 diff --git a/config/piledriver/bli_config.h b/config/piledriver/bli_config.h index 6bd2262b50..dce91516da 100644 --- a/config/piledriver/bli_config.h +++ b/config/piledriver/bli_config.h @@ -36,148 +36,11 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. -#define BLIS_INT_TYPE_SIZE 64 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 4 - -// Enable multithreading via POSIX threads. //#define BLIS_ENABLE_PTHREADS -// Enable multithreading via OpenMP. #define BLIS_ENABLE_OPENMP - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. -#define BLIS_SIMD_ALIGN_SIZE 16 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. -#define BLIS_BLAS2BLIS_INT_TYPE_SIZE 64 - -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - +#define BLIS_SIMD_ALIGN_SIZE 16 #endif diff --git a/config/pnacl/bli_config.h b/config/pnacl/bli_config.h index a05f8d0827..7537125400 100644 --- a/config/pnacl/bli_config.h +++ b/config/pnacl/bli_config.h @@ -36,142 +36,9 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. #define BLIS_INT_TYPE_SIZE 32 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 1 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. #define BLIS_SIMD_ALIGN_SIZE 16 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. #define BLIS_BLAS2BLIS_INT_TYPE_SIZE 32 -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - - #endif diff --git a/config/power7/bli_config.h b/config/power7/bli_config.h index 5a3ce1e1bd..e1725b9b05 100644 --- a/config/power7/bli_config.h +++ b/config/power7/bli_config.h @@ -36,139 +36,6 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. -#define BLIS_INT_TYPE_SIZE 64 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 24 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS 1 -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. -#define BLIS_SIMD_ALIGN_SIZE 16 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE 16 - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE 16 - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. -#define BLIS_BLAS2BLIS_INT_TYPE_SIZE 64 - -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - diff --git a/config/reference/bli_config.h b/config/reference/bli_config.h index 65463146c7..5195e61c5b 100644 --- a/config/reference/bli_config.h +++ b/config/reference/bli_config.h @@ -36,143 +36,6 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. -#define BLIS_INT_TYPE_SIZE 32 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 1 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. -#define BLIS_SIMD_ALIGN_SIZE 16 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. -#define BLIS_BLAS2BLIS_INT_TYPE_SIZE 32 - -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - - #endif diff --git a/config/sandybridge/bli_config.h b/config/sandybridge/bli_config.h index 1d2a4d6bb2..5f66f6dae4 100644 --- a/config/sandybridge/bli_config.h +++ b/config/sandybridge/bli_config.h @@ -35,42 +35,6 @@ #ifndef BLIS_CONFIG_H #define BLIS_CONFIG_H - -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. -#define BLIS_INT_TYPE_SIZE 64 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 16 - // Enable multithreading via POSIX threads. //#define BLIS_ENABLE_PTHREADS @@ -79,105 +43,4 @@ -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS 2 -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. -#define BLIS_SIMD_ALIGN_SIZE 32 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. -#define BLIS_BLAS2BLIS_INT_TYPE_SIZE 64 - -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - - - #endif diff --git a/config/template/bli_config.h b/config/template/bli_config.h index fc65e5135f..cc8e64b5a2 100644 --- a/config/template/bli_config.h +++ b/config/template/bli_config.h @@ -36,142 +36,6 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. -#define BLIS_INT_TYPE_SIZE 64 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 1 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. -#define BLIS_SIMD_ALIGN_SIZE 16 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. -#define BLIS_BLAS2BLIS_INT_TYPE_SIZE 64 - -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - - #endif + diff --git a/frame/1m/packm/bli_packm_init.c b/frame/1m/packm/bli_packm_init.c index af8a3c32f5..caa4e48393 100644 --- a/frame/1m/packm/bli_packm_init.c +++ b/frame/1m/packm/bli_packm_init.c @@ -164,9 +164,6 @@ void bli_packm_init( obj_t* a, } -extern blksz_t* gemm_upanel_a_align; -extern blksz_t* gemm_upanel_b_align; - void bli_packm_init_pack( invdiag_t invert_diag, pack_t schema, packord_t pack_ord_if_up, @@ -327,7 +324,6 @@ void bli_packm_init_pack( invdiag_t invert_diag, { dim_t m_panel; dim_t ps_p, ps_p_orig; - dim_t upanel_a_align; // The panel dimension (for each datatype) should be equal to the // register blocksize in the m dimension. @@ -361,9 +357,6 @@ void bli_packm_init_pack( invdiag_t invert_diag, // Preserve this early panel stride value for use later, if needed. ps_p_orig = ps_p; - // Query the micro-panel alignment for A. - upanel_a_align = bli_blksz_get_def( dt, gemm_upanel_a_align ); - // Here, we adjust the panel stride, if necessary. Remember: ps_p is // always interpreted as being in units of the datatype of the object // which is not necessarily how the micro-panels will be stored. For @@ -374,26 +367,12 @@ void bli_packm_init_pack( invdiag_t invert_diag, if ( bli_is_3mi_packed( schema ) ) { ps_p = ( ps_p * 3 ) / 2; - - // Align the panel stride according to the micro-panel alignment. - ps_p = bli_align_dim_to_size( ps_p, elem_size_p, upanel_a_align ); } else if ( bli_is_3ms_packed( schema ) || bli_is_ro_packed( schema ) || bli_is_io_packed( schema ) || bli_is_rpi_packed( schema ) ) { - // Acquire the element size of the the real projection of the - // current complex datatype. - siz_t elem_size_p_real = elem_size_p / 2; - - // Acquire the micro-panel alignment for the real projection of - // the current complex datatype. - upanel_a_align = bli_blksz_get_def( dt_real, gemm_upanel_a_align ); - - // Align the panel stride according to the micro-panel alignment. - ps_p = bli_align_dim_to_size( ps_p, elem_size_p_real, upanel_a_align ); - // The division by 2 below assumes that ps_p is an even number. // However, it is possible that, at this point, ps_p is an odd. // If it is indeed odd, we nudge it higher. @@ -408,11 +387,6 @@ void bli_packm_init_pack( invdiag_t invert_diag, // actual stored element, we divide the panel_stride by 2. ps_p = ps_p / 2; } - else - { - // Align the panel stride according to the micro-panel alignment. - ps_p = bli_align_dim_to_size( ps_p, elem_size_p, upanel_a_align ); - } // Set the imaginary stride (in units of fundamental elements) for // 3m and 4m (separated or interleaved). We use ps_p_orig since @@ -442,7 +416,6 @@ void bli_packm_init_pack( invdiag_t invert_diag, { dim_t n_panel; dim_t ps_p, ps_p_orig; - dim_t upanel_b_align; // The panel dimension (for each datatype) should be equal to the // register blocksize in the n dimension. @@ -476,9 +449,6 @@ void bli_packm_init_pack( invdiag_t invert_diag, // Preserve this early panel stride value for use later, if needed. ps_p_orig = ps_p; - // Query the micro-panel alignment for B. - upanel_b_align = bli_blksz_get_def( dt, gemm_upanel_b_align ); - // Here, we adjust the panel stride, if necessary. Remember: ps_p is // always interpreted as being in units of the datatype of the object // which is not necessarily how the micro-panels will be stored. For @@ -489,26 +459,12 @@ void bli_packm_init_pack( invdiag_t invert_diag, if ( bli_is_3mi_packed( schema ) ) { ps_p = ( ps_p * 3 ) / 2; - - // Align the panel stride according to the micro-panel alignment. - ps_p = bli_align_dim_to_size( ps_p, elem_size_p, upanel_b_align ); } else if ( bli_is_3ms_packed( schema ) || bli_is_ro_packed( schema ) || bli_is_io_packed( schema ) || bli_is_rpi_packed( schema ) ) { - // Acquire the element size of the the real projection of the - // current complex datatype. - siz_t elem_size_p_real = elem_size_p / 2; - - // Acquire the micro-panel alignment for the real projection of - // the current complex datatype. - upanel_b_align = bli_blksz_get_def( dt_real, gemm_upanel_b_align ); - - // Align the panel stride according to the micro-panel alignment. - ps_p = bli_align_dim_to_size( ps_p, elem_size_p_real, upanel_b_align ); - // The division by 2 below assumes that ps_p is an even number. // However, it is possible that, at this point, ps_p is an odd. // If it is indeed odd, we nudge it higher. @@ -523,11 +479,6 @@ void bli_packm_init_pack( invdiag_t invert_diag, // actual stored element, we divide the panel_stride by 2. ps_p = ps_p / 2; } - else - { - // Align the panel stride according to the micro-panel alignment. - ps_p = bli_align_dim_to_size( ps_p, elem_size_p, upanel_b_align ); - } // Set the imaginary stride (in units of fundamental elements) for // 3m and 4m (separated or interleaved). We use ps_p_orig since diff --git a/frame/3/gemm/bli_gemm_cntl.c b/frame/3/gemm/bli_gemm_cntl.c index 12e28f4511..bf3e60b611 100644 --- a/frame/3/gemm/bli_gemm_cntl.c +++ b/frame/3/gemm/bli_gemm_cntl.c @@ -43,9 +43,6 @@ blksz_t* gemm_mr; blksz_t* gemm_nr; blksz_t* gemm_kr; -blksz_t* gemm_upanel_a_align; -blksz_t* gemm_upanel_b_align; - func_t* gemm_ukrs; func_t* gemm_ref_ukrs; @@ -100,21 +97,6 @@ void bli_gemm_cntl_init() BLIS_DEFAULT_KR_Z, BLIS_PACKDIM_KR_Z ); - // Create objects for micro-panel alignment (in bytes). - gemm_upanel_a_align - = - bli_blksz_obj_create( BLIS_UPANEL_A_ALIGN_SIZE_S, 0, - BLIS_UPANEL_A_ALIGN_SIZE_D, 0, - BLIS_UPANEL_A_ALIGN_SIZE_C, 0, - BLIS_UPANEL_A_ALIGN_SIZE_Z, 0 ); - gemm_upanel_b_align - = - bli_blksz_obj_create( BLIS_UPANEL_B_ALIGN_SIZE_S, 0, - BLIS_UPANEL_B_ALIGN_SIZE_D, 0, - BLIS_UPANEL_B_ALIGN_SIZE_C, 0, - BLIS_UPANEL_B_ALIGN_SIZE_Z, 0 ); - - // Attach the register blksz_t objects as blocksize multiples to the cache // blksz_t objects. bli_blksz_obj_attach_mult_to( gemm_mr, gemm_mc ); @@ -249,9 +231,6 @@ void bli_gemm_cntl_finalize() bli_blksz_obj_free( gemm_nr ); bli_blksz_obj_free( gemm_kr ); - bli_blksz_obj_free( gemm_upanel_a_align ); - bli_blksz_obj_free( gemm_upanel_b_align ); - bli_func_obj_free( gemm_ukrs ); bli_func_obj_free( gemm_ref_ukrs ); diff --git a/frame/base/bli_info.c b/frame/base/bli_info.c index 84b863948b..941a92c3b5 100644 --- a/frame/base/bli_info.c +++ b/frame/base/bli_info.c @@ -57,16 +57,12 @@ char* bli_info_get_int_type_size_str( void ) { return bli_int_type_size_s gint_t bli_info_get_int_type_size( void ) { return BLIS_INT_TYPE_SIZE; } gint_t bli_info_get_num_fp_types( void ) { return BLIS_NUM_FP_TYPES; } gint_t bli_info_get_max_type_size( void ) { return BLIS_MAX_TYPE_SIZE; } -gint_t bli_info_get_max_num_threads( void ) { return BLIS_MAX_NUM_THREADS; } -gint_t bli_info_get_num_mc_x_kc_blocks( void ) { return BLIS_NUM_MC_X_KC_BLOCKS; } -gint_t bli_info_get_num_kc_x_nc_blocks( void ) { return BLIS_NUM_KC_X_NC_BLOCKS; } -gint_t bli_info_get_num_mc_x_nc_blocks( void ) { return BLIS_NUM_MC_X_NC_BLOCKS; } -gint_t bli_info_get_max_preload_byte_offset( void ) { return BLIS_MAX_PRELOAD_BYTE_OFFSET; } gint_t bli_info_get_simd_align_size( void ) { return BLIS_SIMD_ALIGN_SIZE; } +gint_t bli_info_get_page_size( void ) { return BLIS_PAGE_SIZE; } gint_t bli_info_get_stack_buf_align_size( void ) { return BLIS_STACK_BUF_ALIGN_SIZE; } gint_t bli_info_get_heap_addr_align_size( void ) { return BLIS_HEAP_ADDR_ALIGN_SIZE; } gint_t bli_info_get_heap_stride_align_size( void ) { return BLIS_HEAP_STRIDE_ALIGN_SIZE; } -gint_t bli_info_get_contig_addr_align_size( void ) { return BLIS_CONTIG_ADDR_ALIGN_SIZE; } +gint_t bli_info_get_pool_addr_align_size( void ) { return BLIS_POOL_ADDR_ALIGN_SIZE; } gint_t bli_info_get_enable_stay_auto_init( void ) { #ifdef BLIS_ENABLE_STAY_AUTO_INITIALIZED @@ -83,6 +79,14 @@ gint_t bli_info_get_enable_blas2blis( void ) return 0; #endif } +gint_t bli_info_get_enable_cblas( void ) +{ +#ifdef BLIS_ENABLE_CBLAS + return 1; +#else + return 0; +#endif +} gint_t bli_info_get_blas2blis_int_type_size( void ) { return BLIS_BLAS2BLIS_INT_TYPE_SIZE; } @@ -112,41 +116,6 @@ gint_t bli_info_get_default_kr( opid_t oper, num_t dt ) { return bli_bsv_get_ava gint_t bli_info_get_packdim_mr( opid_t oper, num_t dt ) { return bli_bsv_get_avail_blksz_max_dt( BLIS_MR, oper, dt ); } gint_t bli_info_get_packdim_nr( opid_t oper, num_t dt ) { return bli_bsv_get_avail_blksz_max_dt( BLIS_NR, oper, dt ); } -// -- Micro-panel alignment -- - -extern blksz_t* gemm_upanel_a_align; -extern blksz_t* gemm_upanel_b_align; - -// Micro-panel alignment of A - -gint_t bli_info_get_upanel_a_align_size( num_t dt ) -{ - if ( bli_is_float ( dt ) ) return bli_info_get_upanel_a_align_size_s(); - else if ( bli_is_double ( dt ) ) return bli_info_get_upanel_a_align_size_d(); - else if ( bli_is_scomplex( dt ) ) return bli_info_get_upanel_a_align_size_c(); - else if ( bli_is_dcomplex( dt ) ) return bli_info_get_upanel_a_align_size_z(); - else return 0; -} -gint_t bli_info_get_upanel_a_align_size_s( void ) { bli_init(); return bli_blksz_get_def( BLIS_FLOAT, gemm_upanel_a_align ); } -gint_t bli_info_get_upanel_a_align_size_d( void ) { bli_init(); return bli_blksz_get_def( BLIS_DOUBLE, gemm_upanel_a_align ); } -gint_t bli_info_get_upanel_a_align_size_c( void ) { bli_init(); return bli_blksz_get_def( BLIS_SCOMPLEX, gemm_upanel_a_align ); } -gint_t bli_info_get_upanel_a_align_size_z( void ) { bli_init(); return bli_blksz_get_def( BLIS_DCOMPLEX, gemm_upanel_a_align ); } - -// Micro-panel alignment of B - -gint_t bli_info_get_upanel_b_align_size( num_t dt ) -{ - if ( bli_is_float ( dt ) ) return bli_info_get_upanel_b_align_size_s(); - else if ( bli_is_double ( dt ) ) return bli_info_get_upanel_b_align_size_d(); - else if ( bli_is_scomplex( dt ) ) return bli_info_get_upanel_b_align_size_c(); - else if ( bli_is_dcomplex( dt ) ) return bli_info_get_upanel_b_align_size_z(); - else return 0; -} -gint_t bli_info_get_upanel_b_align_size_s( void ) { bli_init(); return bli_blksz_get_def( BLIS_FLOAT, gemm_upanel_b_align ); } -gint_t bli_info_get_upanel_b_align_size_d( void ) { bli_init(); return bli_blksz_get_def( BLIS_DOUBLE, gemm_upanel_b_align ); } -gint_t bli_info_get_upanel_b_align_size_c( void ) { bli_init(); return bli_blksz_get_def( BLIS_SCOMPLEX, gemm_upanel_b_align ); } -gint_t bli_info_get_upanel_b_align_size_z( void ) { bli_init(); return bli_blksz_get_def( BLIS_DCOMPLEX, gemm_upanel_b_align ); } - // -- Level-2 cache blocksizes -- @@ -268,9 +237,9 @@ char* bli_info_get_trsm_u_ukr_impl_string( ind_t method, num_t dt ) // -- bli_mem_pool_macro_defs.h ------------------------------------------------ -gint_t bli_info_get_mk_pool_size( void ) { return BLIS_MK_POOL_SIZE; } -gint_t bli_info_get_kn_pool_size( void ) { return BLIS_KN_POOL_SIZE; } -gint_t bli_info_get_mn_pool_size( void ) { return BLIS_MN_POOL_SIZE; } +gint_t bli_info_get_mk_pool_size( void ) { return bli_mem_pool_size( BLIS_BUFFER_FOR_A_BLOCK ); } +gint_t bli_info_get_kn_pool_size( void ) { return bli_mem_pool_size( BLIS_BUFFER_FOR_B_PANEL ); } +gint_t bli_info_get_mn_pool_size( void ) { return bli_mem_pool_size( BLIS_BUFFER_FOR_C_PANEL ); } diff --git a/frame/base/bli_info.h b/frame/base/bli_info.h index cf8437c41e..7d6d80d0a3 100644 --- a/frame/base/bli_info.h +++ b/frame/base/bli_info.h @@ -44,18 +44,15 @@ char* bli_info_get_int_type_size_str( void ); gint_t bli_info_get_int_type_size( void ); gint_t bli_info_get_num_fp_types( void ); gint_t bli_info_get_max_type_size( void ); -gint_t bli_info_get_max_num_threads( void ); -gint_t bli_info_get_num_mc_x_kc_blocks( void ); -gint_t bli_info_get_num_kc_x_nc_blocks( void ); -gint_t bli_info_get_num_mc_x_nc_blocks( void ); -gint_t bli_info_get_max_preload_byte_offset( void ); gint_t bli_info_get_simd_align_size( void ); +gint_t bli_info_get_page_size( void ); gint_t bli_info_get_stack_buf_align_size( void ); gint_t bli_info_get_heap_addr_align_size( void ); gint_t bli_info_get_heap_stride_align_size( void ); -gint_t bli_info_get_contig_addr_align_size( void ); +gint_t bli_info_get_pool_addr_align_size( void ); gint_t bli_info_get_enable_stay_auto_init( void ); gint_t bli_info_get_enable_blas2blis( void ); +gint_t bli_info_get_enable_cblas( void ); gint_t bli_info_get_blas2blis_int_type_size( void ); @@ -84,22 +81,6 @@ gint_t bli_info_get_default_nr( opid_t oper, num_t dt ); gint_t bli_info_get_packdim_mr( opid_t oper, num_t dt ); gint_t bli_info_get_packdim_nr( opid_t oper, num_t dt ); -// -- Micro-panel alignment for A -- - -gint_t bli_info_get_upanel_a_align_size( num_t dt ); -gint_t bli_info_get_upanel_a_align_size_s( void ); -gint_t bli_info_get_upanel_a_align_size_d( void ); -gint_t bli_info_get_upanel_a_align_size_c( void ); -gint_t bli_info_get_upanel_a_align_size_z( void ); - -// -- Micro-panel alignment for B -- - -gint_t bli_info_get_upanel_b_align_size( num_t dt ); -gint_t bli_info_get_upanel_b_align_size_s( void ); -gint_t bli_info_get_upanel_b_align_size_d( void ); -gint_t bli_info_get_upanel_b_align_size_c( void ); -gint_t bli_info_get_upanel_b_align_size_z( void ); - // -- Level-2 cache blocksizes -- diff --git a/frame/base/bli_init.c b/frame/base/bli_init.c index 7a0c0f39b1..6e793fa404 100644 --- a/frame/base/bli_init.c +++ b/frame/base/bli_init.c @@ -63,41 +63,44 @@ err_t bli_init( void ) // reasons), the conditional test below MUST be within the critical // section to prevent a race condition of the type described above. - // BEGIN CRITICAL SECTION #ifdef BLIS_ENABLE_OPENMP _Pragma( "omp critical (init)" ) #endif #ifdef BLIS_ENABLE_PTHREADS pthread_mutex_lock( &initialize_mutex ); #endif - { - // Proceed with initialization only if BLIS is presently uninitialized. - // Since we bli_init() and bli_finalize() use the same named critical - // section, we can be sure that no other thread is either (a) updating - // bli_is_init, or (b) testing bli_is_init within the critical - // section (for the purposes of deciding whether to perform the - // necessary initialization subtasks). - if ( bli_is_init == FALSE ) + // BEGIN CRITICAL SECTION { - // Initialize various sub-APIs. - bli_const_init(); - bli_cntl_init(); - bli_error_init(); - bli_mem_init(); - bli_ind_init(); - bli_thread_init(); - - // After initialization is complete, mark BLIS as initialized. - bli_is_init = TRUE; - - // Only the thread that actually performs the initialization will - // return "success". - r_val = BLIS_SUCCESS; - } - // END CRITICAL SECTION + // Proceed with initialization only if BLIS is presently uninitialized. + // Since we bli_init() and bli_finalize() use the same named critical + // section, we can be sure that no other thread is either (a) updating + // bli_is_init, or (b) testing bli_is_init within the critical section + // (for the purposes of deciding whether to perform the necessary + // initialization subtasks). + if ( bli_is_init == FALSE ) + { + // Initialize various sub-APIs. + bli_const_init(); + bli_cntl_init(); + bli_error_init(); + bli_mem_init(); + bli_ind_init(); + bli_thread_init(); + + // After initialization is complete, mark BLIS as initialized. + bli_is_init = TRUE; + + //bli_mem_init(); + + // Only the thread that actually performs the initialization will + // return "success". + r_val = BLIS_SUCCESS; + } } + // END CRITICAL SECTION + #ifdef BLIS_ENABLE_PTHREADS pthread_mutex_unlock( &initialize_mutex ); #endif @@ -127,41 +130,42 @@ err_t bli_finalize( void ) // reasons), the conditional test below MUST be within the critical // section to prevent a race condition of the type described above. - // BEGIN CRITICAL SECTION #ifdef BLIS_ENABLE_OPENMP _Pragma( "omp critical (init)" ) #endif #ifdef BLIS_ENABLE_PTHREADS pthread_mutex_lock( &initialize_mutex ); #endif - { - // Proceed with finalization only if BLIS is presently initialized. - // Since we bli_init() and bli_finalize() use the same named critical - // section, we can be sure that no other thread is either (a) updating - // bli_is_init, or (b) testing bli_is_init within the critical - // section (for the purposes of deciding whether to perform the - // necessary finalization subtasks). - if ( bli_is_init == TRUE ) + // BEGIN CRITICAL SECTION { - // Finalize various sub-APIs. - bli_const_finalize(); - bli_cntl_finalize(); - bli_error_finalize(); - bli_mem_finalize(); - bli_ind_finalize(); - bli_thread_finalize(); - - // After finalization is complete, mark BLIS as uninitialized. - bli_is_init = FALSE; - - // Only the thread that actually performs the finalization will - // return "success". - r_val = BLIS_SUCCESS; - } - // END CRITICAL SECTION + // Proceed with finalization only if BLIS is presently initialized. + // Since we bli_init() and bli_finalize() use the same named critical + // section, we can be sure that no other thread is either (a) updating + // bli_is_init, or (b) testing bli_is_init within the critical section + // (for the purposes of deciding whether to perform the necessary + // finalization subtasks). + if ( bli_is_init == TRUE ) + { + // Finalize various sub-APIs. + bli_const_finalize(); + bli_cntl_finalize(); + bli_error_finalize(); + bli_mem_finalize(); + bli_ind_finalize(); + bli_thread_finalize(); + + // After finalization is complete, mark BLIS as uninitialized. + bli_is_init = FALSE; + + // Only the thread that actually performs the finalization will + // return "success". + r_val = BLIS_SUCCESS; + } } + // END CRITICAL SECTION + #ifdef BLIS_ENABLE_PTHREADS pthread_mutex_unlock( &initialize_mutex ); #endif @@ -178,6 +182,8 @@ bool_t bli_is_initialized( void ) return bli_is_init; } +// ----------------------------------------------------------------------------- + void bli_init_auto( err_t* init_result ) { *init_result = bli_init(); diff --git a/frame/base/bli_mem.c b/frame/base/bli_mem.c index 6f13d3c1a6..81a7a2c3b6 100644 --- a/frame/base/bli_mem.c +++ b/frame/base/bli_mem.c @@ -34,101 +34,71 @@ #include "blis.h" -static bool_t bli_mem_is_init = FALSE; - #ifdef BLIS_ENABLE_PTHREADS pthread_mutex_t mem_manager_mutex = PTHREAD_MUTEX_INITIALIZER; #endif // Declare one memory pool structure for each block size/shape we want to // be able to allocate. - static pool_t pools[3]; -// Physically contiguous memory for each pool. -// -// Generally speaking, the pool sizes are computed in a sub-header of blis.h -// as follows: -// -// BLIS_MK_POOL_SIZE = BLIS_MAXIMUM_MC_? * BLIS_MAXIMUM_KC_? * BLIS_SIZEOF_? -// -// where "?" is the datatype that results in the largest pool size. The -// constants BLIS_KN_POOL_SIZE and BLIS_MN_POOL_SIZE are computed in a -// similar manner. All constants are computed with appropriate "padding" -// to ensure enough space given the alignments required by bli_config.h. -// - -static void* pool_mk_blk_ptrs[ BLIS_NUM_MC_X_KC_BLOCKS ]; -static char pool_mk_mem[ BLIS_MK_POOL_SIZE ]; - -static void* pool_kn_blk_ptrs[ BLIS_NUM_KC_X_NC_BLOCKS ]; -static char pool_kn_mem[ BLIS_KN_POOL_SIZE ]; - -static void* pool_mn_blk_ptrs[ BLIS_NUM_MC_X_NC_BLOCKS ]; -static char pool_mn_mem[ BLIS_MN_POOL_SIZE ]; - - - +// ----------------------------------------------------------------------------- void bli_mem_acquire_m( siz_t req_size, packbuf_t buf_type, mem_t* mem ) { - siz_t block_size; - dim_t pool_index; pool_t* pool; - void** block_ptrs; - void* block; - gint_t i; + pblk_t* pblk; + dim_t pi; + siz_t block_size; + // Make sure the API is initialized. + bli_mem_init(); if ( buf_type == BLIS_BUFFER_FOR_GEN_USE ) { // For general-use buffer requests, such as those used by level-2 - // operations, using bli_malloc() is sufficient, since using - // physically contiguous memory is not as important there. - block = bli_malloc( req_size ); + // operations, using bli_malloc() is sufficient. + void* buf_sys = bli_malloc( req_size ); // Initialize the mem_t object with: // - the address of the memory block, // - the buffer type (a packbuf_t value), and // - the size of the requested region. // NOTE: We do not initialize the pool field since this block did not - // come from a contiguous memory pool. - bli_mem_set_buffer( block, mem ); + // come from a memory pool. + bli_mem_set_buffer( buf_sys, mem ); + bli_mem_set_buf_sys( buf_sys, mem ); bli_mem_set_buf_type( buf_type, mem ); bli_mem_set_size( req_size, mem ); } else { // This branch handles cases where the memory block needs to come - // from one of the contiguous memory pools. + // from an internal memory pool, in which blocks are allocated once + // and then recycled. // Map the requested packed buffer type to a zero-based index, which // we then use to select the corresponding memory pool. - pool_index = bli_packbuf_index( buf_type ); - pool = &pools[ pool_index ]; + pi = bli_packbuf_index( buf_type ); + pool = &pools[ pi ]; // Unconditionally perform error checking on the memory pool. { err_t e_val; // Make sure that the requested matrix size fits inside of a block - // of the corresponding pool. + // of the corresponding pool. If it does not, the pool was somehow + // initialized improperly. e_val = bli_check_requested_block_size_for_pool( req_size, pool ); bli_check_error_code( e_val ); - - // Make sure that the pool contains at least one block to check out - // to the thread. - e_val = bli_check_if_exhausted_pool( pool ); - bli_check_error_code( e_val ); } - // Access the block pointer array from the memory pool data structure. - block_ptrs = bli_pool_block_ptrs( pool ); - + // Extract the address of the pblk_t struct within the mem_t. + pblk = bli_mem_pblk( mem ); #ifdef BLIS_ENABLE_OPENMP _Pragma( "omp critical (mem)" ) @@ -139,20 +109,13 @@ void bli_mem_acquire_m( siz_t req_size, // BEGIN CRITICAL SECTION { - // Query the index of the contiguous memory block that resides at the - // "top" of the pool. - i = bli_pool_top_index( pool ); - - // Extract the address of the top block from the block pointer array. - block = block_ptrs[i]; - - // Clear the entry from the block pointer array. (This is actually not - // necessary.) - //block_ptrs[i] = NULL; - - // Decrement the top of the memory pool. - bli_pool_dec_top_index( pool ); - + // Checkout a block from the pool. If the pool is exhausted, + // either because it is still empty or because all blocks have + // been checked out already, additional blocks will be allocated + // automatically, as-needed. Note that the addresses are stored + // directly into the mem_t struct since pblk is the address of + // the struct's pblk_t field. + bli_pool_checkout_block( pblk, pool ); } // END CRITICAL SECTION @@ -162,16 +125,17 @@ void bli_mem_acquire_m( siz_t req_size, #endif // Query the size of the blocks in the pool so we can store it in the - // mem_t object. + // mem_t object. At this point, it is guaranteed to be at least as + // large as req_size. block_size = bli_pool_block_size( pool ); // Initialize the mem_t object with: - // - the address of the memory block, // - the buffer type (a packbuf_t value), // - the address of the memory pool to which it belongs, and // - the size of the contiguous memory block (NOT the size of the // requested region). - bli_mem_set_buffer( block, mem ); + // The actual addresses (system and aligned) are already stored in + // the mem_t struct's pblk_t field bli_mem_set_buf_type( buf_type, mem ); bli_mem_set_pool( pool, mem ); bli_mem_set_size( block_size, mem ); @@ -183,33 +147,30 @@ void bli_mem_release( mem_t* mem ) { packbuf_t buf_type; pool_t* pool; - void** block_ptrs; - void* block; - gint_t i; + pblk_t* pblk; - // Extract the address of the memory block we are trying to - // release. - block = bli_mem_buffer( mem ); + // Make sure the API is initialized. + bli_mem_init(); // Extract the buffer type so we know what kind of memory was allocated. buf_type = bli_mem_buf_type( mem ); if ( buf_type == BLIS_BUFFER_FOR_GEN_USE ) { + void* buf_sys = bli_mem_buf_sys( mem ); + // For general-use buffers, we allocate with bli_malloc(), and so // here we need to call bli_free(). - bli_free( block ); + bli_free( buf_sys ); } else { - // This branch handles cases where the memory block came from one - // of the contiguous memory pools. - - // Extract the pool from which the block was allocated. + // Extract the address of the pool from which the memory was + // allocated. pool = bli_mem_pool( mem ); - // Extract the block pointer array associated with the pool. - block_ptrs = bli_pool_block_ptrs( pool ); + // Extract the address of the pblk_t struct within the mem_t struct. + pblk = bli_mem_pblk( mem ); #ifdef BLIS_ENABLE_OPENMP _Pragma( "omp critical (mem)" ) @@ -221,14 +182,8 @@ void bli_mem_release( mem_t* mem ) // BEGIN CRITICAL SECTION { - // Increment the top of the memory pool. - bli_pool_inc_top_index( pool ); - - // Query the newly incremented top index. - i = bli_pool_top_index( pool ); - - // Place the address of the block back onto the top of the memory pool. - block_ptrs[i] = block; + // Check the block back into the pool. + bli_pool_checkin_block( pblk, pool ); } // END CRITICAL SECTION @@ -238,16 +193,13 @@ void bli_mem_release( mem_t* mem ) #endif } - - // Clear the mem_t object so that it appears unallocated. We clear: - // - the buffer field, - // - the pool field, and - // - the size field. + // Clear the mem_t object so that it appears unallocated. This clears: + // - the pblk_t struct's fields (ie: the buffer addresses) + // - the pool field + // - the size field // NOTE: We do not clear the buf_type field since there is no // "uninitialized" value for packbuf_t. - bli_mem_set_buffer( NULL, mem ); - bli_mem_set_pool( NULL, mem ); - bli_mem_set_size( 0, mem ); + bli_mem_clear( mem ); } @@ -260,20 +212,44 @@ void bli_mem_acquire_v( siz_t req_size, } - -void bli_mem_init( void ) +siz_t bli_mem_pool_size( packbuf_t buf_type ) { - // If the API is already initialized, return early. - if ( bli_mem_is_initialized() ) return; + siz_t r_val; - dim_t index_a; - dim_t index_b; - dim_t index_c; + if ( buf_type == BLIS_BUFFER_FOR_GEN_USE ) + { + // We don't (yet) track the amount of general-purpose + // memory that is currently allocated. + r_val = 0; + } + else + { + dim_t index; + pool_t* pool; + + // Acquire the pointer to the pool corresponding to the buf_type + // provided. + index = bli_packbuf_index( buf_type ); + pool = &(pools[index]); + + // Compute the pool "size" as the product of the block size + // and the number of blocks in the pool. + r_val = bli_pool_block_size( pool ) * + bli_pool_num_blocks( pool ); + } - // Map each of the packbuf_t values to an index starting at zero. - index_a = bli_packbuf_index( BLIS_BUFFER_FOR_A_BLOCK ); - index_b = bli_packbuf_index( BLIS_BUFFER_FOR_B_PANEL ); - index_c = bli_packbuf_index( BLIS_BUFFER_FOR_C_PANEL ); + return r_val; +} + +// ----------------------------------------------------------------------------- + +static bool_t bli_mem_is_init = FALSE; + +void bli_mem_init( void ) +{ + // If the initialization flag is TRUE, we know the API is already + // initialized, so we can return early. + if ( bli_mem_is_init == TRUE ) return; #ifdef BLIS_ENABLE_OPENMP _Pragma( "omp critical (mem)" ) @@ -284,91 +260,32 @@ void bli_mem_init( void ) // BEGIN CRITICAL SECTION { + // Here, we test the initialization flag again. NOTE: THIS IS NOT + // REDUNDANT. This additional test is needed so that other threads + // that may be waiting to acquire the lock do not perform any + // initialization actions once they are finally allowed into this + // critical section. + if ( bli_mem_is_init == FALSE ) + { + // Initialize the memory pools. + bli_mem_init_pools(); - // Initialize contiguous memory pool for MC x KC blocks. - bli_mem_init_pool( pool_mk_mem, - BLIS_MK_BLOCK_SIZE, - BLIS_NUM_MC_X_KC_BLOCKS, - pool_mk_blk_ptrs, - &pools[ index_a ] ); - - // Initialize contiguous memory pool for KC x NC blocks. - bli_mem_init_pool( pool_kn_mem, - BLIS_KN_BLOCK_SIZE, - BLIS_NUM_KC_X_NC_BLOCKS, - pool_kn_blk_ptrs, - &pools[ index_b ] ); - - // Initialize contiguous memory pool for MC x NC blocks. - bli_mem_init_pool( pool_mn_mem, - BLIS_MN_BLOCK_SIZE, - BLIS_NUM_MC_X_NC_BLOCKS, - pool_mn_blk_ptrs, - &pools[ index_c ] ); - + // After initialization, mark the API as initialized. + bli_mem_is_init = TRUE; + } } // END CRITICAL SECTION #ifdef BLIS_ENABLE_PTHREADS pthread_mutex_unlock( &mem_manager_mutex ); #endif - - // Mark API as initialized. - bli_mem_is_init = TRUE; } - -void bli_mem_init_pool( char* pool_mem, - siz_t block_size, - dim_t num_blocks, - void** block_ptrs, - pool_t* pool ) -{ - const siz_t align_size = BLIS_CONTIG_ADDR_ALIGN_SIZE; - dim_t i; - - // If the pool starting address is not already aligned, advance it - // accordingly. - if ( bli_is_unaligned_to( ( uintptr_t )pool_mem, ( uintptr_t )align_size ) ) - { - // Notice that this works even if the alignment is not a power of two. - pool_mem += ( ( uintptr_t )align_size - - ( ( uintptr_t )pool_mem % align_size ) ); - } - - // Step through the memory pool, beginning with the aligned address - // determined above, assigning pointers to the beginning of each block_size - // bytes to the ith element of the block_ptrs array. - for ( i = 0; i < num_blocks; ++i ) - { - // Save the address of pool, which is guaranteed to be aligned. - block_ptrs[i] = pool_mem; - - // Advance pool by one block. - pool_mem += block_size; - - // Advance pool a bit further if needed in order to get to the - // beginning of an alignment boundary. - if ( bli_is_unaligned_to( ( uintptr_t )pool_mem, ( uintptr_t )align_size ) ) - { - pool_mem += ( ( uintptr_t )align_size - - ( ( uintptr_t )pool_mem % align_size ) ); - } - } - - // Now that we have initialized the array of pointers to the individual - // blocks in the pool, we initialize a pool_t data structure so that we - // can easily manage this pool. - bli_pool_init( num_blocks, - block_size, - block_ptrs, - pool ); -} - - - void bli_mem_finalize( void ) { + // If the initialization flag is FALSE, we know the API is already + // uninitialized, so we can return early. + if ( bli_mem_is_init == FALSE ) return; #ifdef BLIS_ENABLE_OPENMP _Pragma( "omp critical (mem)" ) @@ -379,20 +296,25 @@ void bli_mem_finalize( void ) // BEGIN CRITICAL SECTION { - // Do nothing. + // Here, we test the initialization flag again. NOTE: THIS IS NOT + // REDUNDANT. This additional test is needed so that other threads + // that may be waiting to acquire the lock do not perform any + // finalization actions once they are finally allowed into this + // critical section. + if ( bli_mem_is_init == TRUE ) + { + // Finalize the memory pools. + bli_mem_finalize_pools(); + + // After finalization, mark the API as uninitialized. + bli_mem_is_init = FALSE; + } } // END CRITICAL SECTION #ifdef BLIS_ENABLE_PTHREADS pthread_mutex_unlock( &mem_manager_mutex ); #endif - -#ifdef BLIS_ENABLE_PTHREADS - pthread_mutex_destroy( &mem_manager_mutex ); -#endif - - // Mark API as uninitialized. - bli_mem_is_init = FALSE; } bool_t bli_mem_is_initialized( void ) @@ -400,3 +322,203 @@ bool_t bli_mem_is_initialized( void ) return bli_mem_is_init; } +// ----------------------------------------------------------------------------- + +void bli_mem_init_pools( void ) +{ + // Map each of the packbuf_t values to an index starting at zero. + dim_t index_a = bli_packbuf_index( BLIS_BUFFER_FOR_A_BLOCK ); + dim_t index_b = bli_packbuf_index( BLIS_BUFFER_FOR_B_PANEL ); + dim_t index_c = bli_packbuf_index( BLIS_BUFFER_FOR_C_PANEL ); + + dim_t num_blocks_a = 0; + dim_t num_blocks_b = 0; + dim_t num_blocks_c = 0; + + siz_t align_size = BLIS_POOL_ADDR_ALIGN_SIZE; + + siz_t block_size_a = 0; + siz_t block_size_b = 0; + siz_t block_size_c = 0; + + ind_t im; + num_t dt; + + // Compute pool block sizes for datatype and each implemented + // method and find the maximum size for each pool. This is done + // so that new pools do not need to be allocated if the user + // switches datatypes or methods later on. + for ( im = 0; im < BLIS_NUM_IND_METHODS; ++im ) + { + for ( dt = BLIS_DT_LO; dt <= BLIS_DT_HI; ++dt ) + { + // Avoid considering induced methods for real datatypes. + if ( bli_is_complex( dt ) || im == BLIS_NAT ) + { + siz_t bs_a, bs_b, bs_c; +//printf( "bli_mem_init_pools: considering dt %lu im %lu\n", dt, im ); + bli_mem_compute_pool_block_sizes_dt( dt, im, + &bs_a, + &bs_b, + &bs_c ); + +//printf( "bli_mem_init_pools: bs a b c = %lu %lu %lu\n", bs_a, bs_b, bs_c ); + block_size_a = bli_max( bs_a, block_size_a ); + block_size_b = bli_max( bs_b, block_size_b ); + block_size_c = bli_max( bs_c, block_size_c ); + } + } + } + + // Initialize the memory pools for A, B, and C. + bli_pool_init( num_blocks_a, block_size_a, align_size, &pools[ index_a ] ); + bli_pool_init( num_blocks_b, block_size_b, align_size, &pools[ index_b ] ); + bli_pool_init( num_blocks_c, block_size_c, align_size, &pools[ index_c ] ); +} + +void bli_mem_finalize_pools( void ) +{ + // Map each of the packbuf_t values to an index starting at zero. + dim_t index_a = bli_packbuf_index( BLIS_BUFFER_FOR_A_BLOCK ); + dim_t index_b = bli_packbuf_index( BLIS_BUFFER_FOR_B_PANEL ); + dim_t index_c = bli_packbuf_index( BLIS_BUFFER_FOR_C_PANEL ); + + // Finalize the memory pools for A, B, and C. + bli_pool_finalize( &pools[ index_a ] ); + bli_pool_finalize( &pools[ index_b ] ); + bli_pool_finalize( &pools[ index_c ] ); +} + +// ----------------------------------------------------------------------------- + +void bli_mem_compute_pool_block_sizes_dt( num_t dt, + ind_t method, + siz_t* bs_a, + siz_t* bs_b, + siz_t* bs_c ) +{ + siz_t size_dt = bli_datatype_size( dt ); + + blksz_t* mr; + blksz_t* nr; + + blksz_t* mc; + blksz_t* kc; + blksz_t* nc; + + dim_t mr_dt; + dim_t nr_dt; + dim_t max_mnr_dt; + + dim_t mc_max_dt; + dim_t kc_max_dt; + dim_t nc_max_dt; + + dim_t packmr_dt, packnr_dt; + dim_t max_packmnr_dt; + + dim_t scale_num_dt; + dim_t scale_den_dt; + + dim_t pool_mc_dt, left_mc_dt; + dim_t pool_nc_dt, left_nc_dt; + dim_t pool_kc_dt; + + // + // Find the larger of the two register blocksizes. + // + + // Query the mr and nr blksz_t objects for the given method of + // execution. + mr = bli_bsv_get_blksz( BLIS_MR, method ); + nr = bli_bsv_get_blksz( BLIS_NR, method ); + + // Extract the mr and nr values specific to the current datatype. + mr_dt = bli_blksz_get_def( dt, mr ); + nr_dt = bli_blksz_get_def( dt, nr ); + + // Find the maximum of mr and nr. + max_mnr_dt = bli_max( mr_dt, nr_dt ); + + // + // Define local maximum cache blocksizes. + // + + // Query the mc, kc, and nc blksz_t objects for native execution. + mc = bli_bsv_get_blksz( BLIS_MC, method ); + kc = bli_bsv_get_blksz( BLIS_KC, method ); + nc = bli_bsv_get_blksz( BLIS_NC, method ); + + // Extract the maximum mc, kc, and nc values specific to the current + // datatype. + mc_max_dt = bli_blksz_get_max( dt, mc ); + kc_max_dt = bli_blksz_get_max( dt, kc ); + nc_max_dt = bli_blksz_get_max( dt, nc ); + + // Add max(mr,nr) to kc to make room for the nudging of kc at + // runtime to be a multiple of mr or nr for triangular operations + // trmm, trmm3, and trsm. + kc_max_dt += max_mnr_dt; + + // + // Compute scaling factors. + // + + // Compute integer scaling factors (numerator and denominator) used + // to account for situations when the packing register blocksizes are + // larger than the regular register blocksizes. + + // In order to compute the scaling factors, we first have to determine + // whether ( packmr / mr ) is greater than ( packnr / nr ). This is + // needed ONLY because the amount of space allocated for a block of A + // and a panel of B needs to be such that MR and NR can be swapped (ie: + // A is packed with NR and B is packed with MR). This transformation is + // needed for right-side trsm when inducing an algorithm that (a) has + // favorable access patterns for column-stored C and (b) allows the + // macro-kernel to reuse the existing left-side fused gemmtrsm micro- + // kernels. We avoid integer division by cross-multiplying: + // + // ( packmr / mr ) >= ( packnr / nr ) + // ( packmr / mr ) * nr >= packnr + // packmr * nr >= packnr * mr + // + // So, if packmr * nr >= packnr * mr, then we will use packmr and mr as + // our scaling factors. Otherwise, we'll use packnr and nr. + + packmr_dt = bli_blksz_get_max( dt, mr ); + packnr_dt = bli_blksz_get_max( dt, nr ); + + if ( packmr_dt * nr_dt >= + packnr_dt * mr_dt ) { scale_num_dt = packmr_dt; + scale_den_dt = mr_dt; } + else { scale_num_dt = packnr_dt; + scale_den_dt = nr_dt; } + + // + // Compute pool block dimensions. + // + + pool_mc_dt = ( mc_max_dt * scale_num_dt ) / scale_den_dt; + left_mc_dt = ( mc_max_dt * scale_num_dt ) % scale_den_dt; + + pool_nc_dt = ( nc_max_dt * scale_num_dt ) / scale_den_dt; + left_nc_dt = ( nc_max_dt * scale_num_dt ) % scale_den_dt; + + pool_kc_dt = ( kc_max_dt ); + + if ( left_mc_dt > 0 ) pool_mc_dt += 1; + if ( left_nc_dt > 0 ) pool_nc_dt += 1; + + // + // Compute pool block sizes + // + + // We add an extra micro-panel of space to the block sizes for A and B + // just to be sure any pre-loading performed by the micro-kernel does + // not cause a segmentation fault. + max_packmnr_dt = bli_max( packmr_dt, packnr_dt ); + + *bs_a = ( pool_mc_dt + max_packmnr_dt ) * pool_kc_dt * size_dt; + *bs_b = ( pool_nc_dt + max_packmnr_dt ) * pool_kc_dt * size_dt; + *bs_c = ( pool_mc_dt ) * pool_nc_dt * size_dt; +} diff --git a/frame/base/bli_mem.c.prev b/frame/base/bli_mem.c.prev new file mode 100644 index 0000000000..7a16e87327 --- /dev/null +++ b/frame/base/bli_mem.c.prev @@ -0,0 +1,366 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + +#ifdef BLIS_ENABLE_PTHREADS +extern pthread_mutex_t mem_manager_mutex; +#endif + +// Declare one memory pool structure for each block size/shape we want to +// be able to allocate. + +static pool_t pools[3]; + + +// Physically contiguous memory for each pool. +// +// Generally speaking, the pool sizes are computed in a sub-header of blis.h +// as follows: +// +// BLIS_MK_POOL_SIZE = BLIS_MAXIMUM_MC_? * BLIS_MAXIMUM_KC_? * BLIS_SIZEOF_? +// +// where "?" is the datatype that results in the largest pool size. The +// constants BLIS_KN_POOL_SIZE and BLIS_MN_POOL_SIZE are computed in a +// similar manner. All constants are computed with appropriate "padding" +// to ensure enough space given the alignments required by bli_config.h. +// + +static void* pool_mk_blk_ptrs[ BLIS_NUM_MC_X_KC_BLOCKS ]; +static void* pool_kn_blk_ptrs[ BLIS_NUM_KC_X_NC_BLOCKS ]; +static void* pool_mn_blk_ptrs[ BLIS_NUM_MC_X_NC_BLOCKS ]; + +#define BLIS_USE_HEAP + +#ifdef BLIS_USE_HEAP +static char* pool_mk_mem = NULL; +static char* pool_kn_mem = NULL; +static char* pool_mn_mem = NULL; +#else +static char pool_mk_mem[ BLIS_MK_POOL_SIZE ]; +static char pool_kn_mem[ BLIS_KN_POOL_SIZE ]; +static char pool_mn_mem[ BLIS_MN_POOL_SIZE ]; +#endif + + + +void bli_mem_acquire_m( siz_t req_size, + packbuf_t buf_type, + mem_t* mem ) +{ + siz_t block_size; + dim_t pool_index; + pool_t* pool; + void** block_ptrs; + void* block; + gint_t i; + + + if ( buf_type == BLIS_BUFFER_FOR_GEN_USE ) + { + // For general-use buffer requests, such as those used by level-2 + // operations, using bli_malloc() is sufficient, since using + // physically contiguous memory is not as important there. + block = bli_malloc( req_size ); + + // Initialize the mem_t object with: + // - the address of the memory block, + // - the buffer type (a packbuf_t value), and + // - the size of the requested region. + // NOTE: We do not initialize the pool field since this block did not + // come from a contiguous memory pool. + bli_mem_set_buffer( block, mem ); + bli_mem_set_buf_type( buf_type, mem ); + bli_mem_set_size( req_size, mem ); + } + else + { + // This branch handles cases where the memory block needs to come + // from one of the contiguous memory pools. + + // Map the requested packed buffer type to a zero-based index, which + // we then use to select the corresponding memory pool. + pool_index = bli_packbuf_index( buf_type ); + pool = &pools[ pool_index ]; + + // Unconditionally perform error checking on the memory pool. + { + err_t e_val; + + // Make sure that the requested matrix size fits inside of a block + // of the corresponding pool. + e_val = bli_check_requested_block_size_for_pool( req_size, pool ); + bli_check_error_code( e_val ); + + // Make sure that the pool contains at least one block to check out + // to the thread. + e_val = bli_check_if_exhausted_pool( pool ); + bli_check_error_code( e_val ); + } + + // Access the block pointer array from the memory pool data structure. + block_ptrs = bli_pool_block_ptrs( pool ); + + + // BEGIN CRITICAL SECTION +#ifdef BLIS_ENABLE_OPENMP + _Pragma( "omp critical (mem)" ) +#endif +#ifdef BLIS_ENABLE_PTHREADS + pthread_mutex_lock( &mem_manager_mutex ); +#endif + { + + // Query the index of the contiguous memory block that resides at the + // "top" of the pool. + i = bli_pool_top_index( pool ); + + // Extract the address of the top block from the block pointer array. + block = block_ptrs[i]; + + // Clear the entry from the block pointer array. (This is actually not + // necessary.) + //block_ptrs[i] = NULL; + + // Decrement the top of the memory pool. + bli_pool_dec_top_index( pool ); + + + // END CRITICAL SECTION + } +#ifdef BLIS_ENABLE_PTHREADS + pthread_mutex_unlock( &mem_manager_mutex ); +#endif + + // Query the size of the blocks in the pool so we can store it in the + // mem_t object. + block_size = bli_pool_block_size( pool ); + + // Initialize the mem_t object with: + // - the address of the memory block, + // - the buffer type (a packbuf_t value), + // - the address of the memory pool to which it belongs, and + // - the size of the contiguous memory block (NOT the size of the + // requested region). + bli_mem_set_buffer( block, mem ); + bli_mem_set_buf_type( buf_type, mem ); + bli_mem_set_pool( pool, mem ); + bli_mem_set_size( block_size, mem ); + } +} + + +void bli_mem_release( mem_t* mem ) +{ + packbuf_t buf_type; + pool_t* pool; + void** block_ptrs; + void* block; + gint_t i; + + // Extract the address of the memory block we are trying to + // release. + block = bli_mem_buffer( mem ); + + // Extract the buffer type so we know what kind of memory was allocated. + buf_type = bli_mem_buf_type( mem ); + + if ( buf_type == BLIS_BUFFER_FOR_GEN_USE ) + { + // For general-use buffers, we allocate with bli_malloc(), and so + // here we need to call bli_free(). + bli_free( block ); + } + else + { + // This branch handles cases where the memory block came from one + // of the contiguous memory pools. + + // Extract the pool from which the block was allocated. + pool = bli_mem_pool( mem ); + + // Extract the block pointer array associated with the pool. + block_ptrs = bli_pool_block_ptrs( pool ); + + + // BEGIN CRITICAL SECTION +#ifdef BLIS_ENABLE_OPENMP + _Pragma( "omp critical (mem)" ) +#endif +#ifdef BLIS_ENABLE_PTHREADS + pthread_mutex_lock( &mem_manager_mutex ); +#endif + { + + // Increment the top of the memory pool. + bli_pool_inc_top_index( pool ); + + // Query the newly incremented top index. + i = bli_pool_top_index( pool ); + + // Place the address of the block back onto the top of the memory pool. + block_ptrs[i] = block; + + + // END CRITICAL SECTION + } +#ifdef BLIS_ENABLE_PTHREADS + pthread_mutex_unlock( &mem_manager_mutex ); +#endif + } + + + // Clear the mem_t object so that it appears unallocated. We clear: + // - the buffer field, + // - the pool field, and + // - the size field. + // NOTE: We do not clear the buf_type field since there is no + // "uninitialized" value for packbuf_t. + bli_mem_set_buffer( NULL, mem ); + bli_mem_set_pool( NULL, mem ); + bli_mem_set_size( 0, mem ); +} + + +void bli_mem_acquire_v( siz_t req_size, + mem_t* mem ) +{ + bli_mem_acquire_m( req_size, + BLIS_BUFFER_FOR_GEN_USE, + mem ); +} + + + +void bli_mem_init() +{ + dim_t index_a; + dim_t index_b; + dim_t index_c; + +#ifdef BLIS_USE_HEAP + pool_mk_mem = bli_malloc( BLIS_MK_POOL_SIZE ); + pool_kn_mem = bli_malloc( BLIS_KN_POOL_SIZE ); + pool_mn_mem = bli_malloc( BLIS_MN_POOL_SIZE ); +#endif + + // Map each of the packbuf_t values to an index starting at zero. + index_a = bli_packbuf_index( BLIS_BUFFER_FOR_A_BLOCK ); + index_b = bli_packbuf_index( BLIS_BUFFER_FOR_B_PANEL ); + index_c = bli_packbuf_index( BLIS_BUFFER_FOR_C_PANEL ); + + // Initialize contiguous memory pool for MC x KC blocks. + bli_mem_init_pool( pool_mk_mem, + BLIS_MK_BLOCK_SIZE, + BLIS_NUM_MC_X_KC_BLOCKS, + pool_mk_blk_ptrs, + &pools[ index_a ] ); + + // Initialize contiguous memory pool for KC x NC blocks. + bli_mem_init_pool( pool_kn_mem, + BLIS_KN_BLOCK_SIZE, + BLIS_NUM_KC_X_NC_BLOCKS, + pool_kn_blk_ptrs, + &pools[ index_b ] ); + + // Initialize contiguous memory pool for MC x NC blocks. + bli_mem_init_pool( pool_mn_mem, + BLIS_MN_BLOCK_SIZE, + BLIS_NUM_MC_X_NC_BLOCKS, + pool_mn_blk_ptrs, + &pools[ index_c ] ); +} + + +void bli_mem_init_pool( char* pool_mem, + siz_t block_size, + dim_t num_blocks, + void** block_ptrs, + pool_t* pool ) +{ + const siz_t align_size = BLIS_CONTIG_ADDR_ALIGN_SIZE; + dim_t i; + + // If the pool starting address is not already aligned, advance it + // accordingly. + if ( bli_is_unaligned_to( ( uintptr_t )pool_mem, ( uintptr_t )align_size ) ) + { + // Notice that this works even if the alignment is not a power of two. + pool_mem += ( ( uintptr_t )align_size - + ( ( uintptr_t )pool_mem % align_size ) ); + } + + // Step through the memory pool, beginning with the aligned address + // determined above, assigning pointers to the beginning of each block_size + // bytes to the ith element of the block_ptrs array. + for ( i = 0; i < num_blocks; ++i ) + { + // Save the address of pool, which is guaranteed to be aligned. + block_ptrs[i] = pool_mem; + + // Advance pool by one block. + pool_mem += block_size; + + // Advance pool a bit further if needed in order to get to the + // beginning of an alignment boundary. + if ( bli_is_unaligned_to( ( uintptr_t )pool_mem, ( uintptr_t )align_size ) ) + { + pool_mem += ( ( uintptr_t )align_size - + ( ( uintptr_t )pool_mem % align_size ) ); + } + } + + // Now that we have initialized the array of pointers to the individual + // blocks in the pool, we initialize a pool_t data structure so that we + // can easily manage this pool. + bli_pool_init( num_blocks, + block_size, + block_ptrs, + pool ); +} + + + +void bli_mem_finalize() +{ + // Nothing to do. + +#ifdef BLIS_USE_HEAP + bli_free( pool_mk_mem ); + bli_free( pool_kn_mem ); + bli_free( pool_mn_mem ); +#endif + +} + diff --git a/frame/base/bli_mem.h b/frame/base/bli_mem.h index 8fc6d85d6f..59572ad41c 100644 --- a/frame/base/bli_mem.h +++ b/frame/base/bli_mem.h @@ -36,6 +36,7 @@ void bli_mem_init( void ); void bli_mem_finalize( void ); bool_t bli_mem_is_initialized( void ); +// ----------------------------------------------------------------------------- void bli_mem_acquire_m( siz_t req_size, packbuf_t buf_type, @@ -46,9 +47,16 @@ void bli_mem_acquire_v( siz_t req_size, void bli_mem_release( mem_t* mem ); -void bli_mem_init_pool( char* pool_mem, - siz_t block_size, - dim_t n_blocks, - void** block_ptrs, - pool_t* pool_struct ); +siz_t bli_mem_pool_size( packbuf_t buf_type ); + +// ----------------------------------------------------------------------------- + +void bli_mem_init_pools( void ); +void bli_mem_finalize_pools( void ); + +void bli_mem_compute_pool_block_sizes_dt( num_t dt, + ind_t method, + siz_t* bs_a, + siz_t* bs_b, + siz_t* bs_c ); diff --git a/frame/base/bli_pool.c b/frame/base/bli_pool.c new file mode 100644 index 0000000000..9c7230faa7 --- /dev/null +++ b/frame/base/bli_pool.c @@ -0,0 +1,344 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + +void bli_pool_init( dim_t num_blocks_init, + siz_t block_size, + siz_t align_size, + pool_t* pool ) +{ + pblk_t* block_ptrs; + dim_t i; + + // Allocate the block_ptrs array. + block_ptrs = bli_malloc( num_blocks_init * sizeof( pblk_t ) ); + + // Allocate and initialize each entry in the block_ptrs array. + for ( i = 0; i < num_blocks_init; ++i ) + { + bli_pool_alloc_block( block_size, align_size, &(block_ptrs[i]) ); + } + + // Initialize the pool_t structure. + bli_pool_set_block_ptrs( block_ptrs, pool ); + bli_pool_set_block_ptrs_len( num_blocks_init, pool ); + bli_pool_set_num_blocks( num_blocks_init, pool ); + bli_pool_set_top_index( 0, pool ); + bli_pool_set_block_size( block_size, pool ); + bli_pool_set_align_size( align_size, pool ); +} + +void bli_pool_finalize( pool_t* pool ) +{ + pblk_t* block_ptrs; + dim_t num_blocks; + dim_t i; + + // NOTE: This implementation assumes that all blocks have been + // checked in by all threads. + + // Query the current block_ptrs array and total number of blocks + // presently allocated. + block_ptrs = bli_pool_block_ptrs( pool ); + num_blocks = bli_pool_num_blocks( pool ); + + // Free the individual blocks. + for ( i = 0; i < num_blocks; ++i ) + { + bli_pool_free_block( &(block_ptrs[i]) ); + } + + // Free the block_ptrs array. + bli_free( block_ptrs ); +} + +void bli_pool_checkout_block( pblk_t* block, pool_t* pool ) +{ + pblk_t* block_ptrs; + dim_t top_index; + + // If the pool is exhausted, add a block. + if ( bli_pool_is_exhausted( pool ) ) + { + bli_pool_grow( 1, pool ); + } + + // At this point, at least one block is guaranteed to be available. + + // Query the current block_ptrs array. + block_ptrs = bli_pool_block_ptrs( pool ); + + // Query the top_index of the pool. + top_index = bli_pool_top_index( pool ); + + // Copy the block at top_index to the caller's pblk_t struct. + //bli_pblk_copy( *(block_ptrs[top_index]), *block ); + *block = block_ptrs[top_index]; + + // Notice that we don't actually need to clear the contents of + // block_ptrs[top_index]. It will get overwritten eventually when + // the block is checked back in. + bli_pblk_clear( &block_ptrs[top_index] ); + + // Increment the pool's top_index. + bli_pool_set_top_index( top_index + 1, pool ); +} + +void bli_pool_checkin_block( pblk_t* block, pool_t* pool ) +{ + pblk_t* block_ptrs; + dim_t top_index; + + // Query the current block_ptrs array. + block_ptrs = bli_pool_block_ptrs( pool ); + + // Query the top_index of the pool. + top_index = bli_pool_top_index( pool ); + + // Copy the caller's pblk_t struct to the block at top_index - 1. + //bli_pblk_copy( *(block_ptrs[top_index-1]), *block ); + block_ptrs[top_index-1] = *block; + + // Decrement the pool's top_index. + bli_pool_set_top_index( top_index - 1, pool ); +} + +void bli_pool_grow( dim_t num_blocks_add, pool_t* pool ) +{ + pblk_t* block_ptrs_cur; + dim_t block_ptrs_len_cur; + dim_t num_blocks_cur; + + pblk_t* block_ptrs_new; + dim_t num_blocks_new; + + siz_t block_size; + siz_t align_size; + dim_t top_index; + + dim_t i; + + // If the requested increase is zero (or negative), return early. + if ( num_blocks_add < 1 ) return; + + // Query the allocated length of the block_ptrs array and also the + // total number of blocks allocated. + block_ptrs_len_cur = bli_pool_block_ptrs_len( pool ); + num_blocks_cur = bli_pool_num_blocks( pool ); + + // Compute the total number of allocated blocks that will exist + // after we grow the pool. + num_blocks_new = num_blocks_cur + num_blocks_add; + + // If the new total number of allocated blocks is larger than the + // allocated length of the block_ptrs array, we need to allocate + // a new (larger) block_ptrs array. + if ( num_blocks_new > block_ptrs_len_cur ) + { + // Query the current block_ptrs array. + block_ptrs_cur = bli_pool_block_ptrs( pool ); + + // Allocate a new block_ptrs array of length num_blocks_new. + block_ptrs_new = bli_malloc( num_blocks_new * sizeof( pblk_t ) ); + + // Query the top_index of the pool. + top_index = bli_pool_top_index( pool ); + + // Copy the contents of the old block_ptrs array to the new/resized + // array. Notice that we can begin with top_index since all entries + // from 0 to top_index-1 have been checked out to threads. + for ( i = top_index; i < num_blocks_cur; ++i ) + { +//printf( "bli_pool_grow: copying from %lu\n", top_index ); + block_ptrs_new[i] = block_ptrs_cur[i]; + } + +//printf( "bli_pool_grow: bp_cur: %p\n", block_ptrs_cur ); + // Free the old block_ptrs array. + bli_free( block_ptrs_cur ); + + // Update the pool_t struct with the new block_ptrs array and + // record its allocated length. + bli_pool_set_block_ptrs( block_ptrs_new, pool ); + bli_pool_set_block_ptrs_len( num_blocks_new, pool ); + } + + // At this point, we are guaranteed to have enough unused elements + // in the block_ptrs array to accommodate an additional num_blocks_add + // blocks. + + // Query the current block_ptrs array (which was possibly just resized). + block_ptrs_cur = bli_pool_block_ptrs( pool ); + + // Query the block size and alignment size of the current pool. + block_size = bli_pool_block_size( pool ); + align_size = bli_pool_align_size( pool ); + + // Allocate the requested additional blocks in the resized array. + for ( i = num_blocks_cur; i < num_blocks_new; ++i ) + { +//printf( "libblis: growing pool, block_size = %lu\n", block_size ); fflush( stdout ); + + bli_pool_alloc_block( block_size, align_size, &(block_ptrs_cur[i]) ); + } + + // Update the pool_t struct with the new number of allocated blocks. + // Notice that top_index remains unchanged, as do the block_size and + // align_size fields. + bli_pool_set_num_blocks( num_blocks_new, pool ); +} + +void bli_pool_shrink( dim_t num_blocks_sub, pool_t* pool ) +{ + pblk_t* block_ptrs; + dim_t num_blocks; + dim_t num_blocks_avail; + dim_t num_blocks_new; + + dim_t top_index; + + dim_t i; + + // Query the total number of blocks presently allocated. + num_blocks = bli_pool_num_blocks( pool ); + + // Query the top_index of the pool. + top_index = bli_pool_top_index( pool ); + + // Compute the number of blocks available to be checked out + // (and thus available for removal). + num_blocks_avail = num_blocks - top_index; + + // If the requested decrease is more than the number of available + // blocks in the pool, only remove the number of blocks available. + if ( num_blocks_avail < num_blocks_sub ) + num_blocks_sub = num_blocks_avail; + + // If the effective requested decrease is zero (or the requested + // decrease was negative), return early. + if ( num_blocks_sub < 1 ) return; + + // Query the current block_ptrs array. + block_ptrs = bli_pool_block_ptrs( pool ); + + // Compute the new total number of blocks. + num_blocks_new = num_blocks - num_blocks_sub; + + // Free the individual blocks. + for ( i = num_blocks_new; i < num_blocks; ++i ) + { + bli_pool_free_block( &(block_ptrs[i]) ); + } + + // Update the pool_t struct. + bli_pool_set_num_blocks( num_blocks_new, pool ); + + // Note that after shrinking the pool, num_blocks < block_ptrs_len. + // This means the pool can grow again by num_blocks_sub before + // a re-allocation of block_ptrs is triggered. +} + +void bli_pool_alloc_block( siz_t block_size, + siz_t align_size, + pblk_t* block ) +{ + void* buf_sys; + void* buf_align; + + // Allocate the block. We add the alignment size to ensure we will + // have enough usable space after alignment. + buf_sys = bli_malloc( block_size + align_size ); + buf_align = buf_sys; + + // Advance the pointer to achieve the necessary alignment, if it + // is not already aligned. + if ( bli_is_unaligned_to( ( uintptr_t )buf_sys, ( uintptr_t )align_size ) ) + { + // Notice that this works even if the alignment is not a power of two. + buf_align += ( ( uintptr_t )align_size - + ( ( uintptr_t )buf_sys % align_size ) ); + } + + // Save the results in the pblk_t structure. + bli_pblk_set_buf_sys( buf_sys, block ); + bli_pblk_set_buf_align( buf_align, block ); +} + +void bli_pool_free_block( pblk_t* block ) +{ + void* buf_sys; + + // Extract the pointer to the block that was originally provided by + // the operating system. + buf_sys = bli_pblk_buf_sys( block ); + + // Free the block. + bli_free( buf_sys ); +} + +void bli_pool_print( pool_t* pool ) +{ + pblk_t* block_ptrs = bli_pool_block_ptrs( pool ); + dim_t block_ptrs_len = bli_pool_block_ptrs_len( pool ); + dim_t top_index = bli_pool_top_index( pool ); + dim_t num_blocks = bli_pool_num_blocks( pool ); + dim_t block_size = bli_pool_block_size( pool ); + dim_t align_size = bli_pool_align_size( pool ); + dim_t i; + + printf( "pool struct ---------------\n" ); + printf( " block_ptrs: %p\n", block_ptrs ); + printf( " block_ptrs_len: %ld\n", block_ptrs_len ); + printf( " top_index: %ld\n", top_index ); + printf( " num_blocks: %ld\n", num_blocks ); + printf( " block_size: %ld\n", block_size ); + printf( " align_size: %ld\n", align_size ); + printf( " pblks sys align\n" ); + for ( i = 0; i < num_blocks; ++i ) + { + printf( " %ld: %p %p\n", i, bli_pblk_buf_sys( &block_ptrs[i] ), + bli_pblk_buf_align( &block_ptrs[i] ) ); + } +} + +void bli_pblk_print( pblk_t* pblk ) +{ + void* buf_sys = bli_pblk_buf_sys( pblk ); + void* buf_align = bli_pblk_buf_align( pblk ); + + printf( "pblk struct ---------------\n" ); + printf( " %p %p\n", buf_sys, buf_align ); +} + diff --git a/frame/include/bli_pool_macro_defs.h b/frame/base/bli_pool.h similarity index 54% rename from frame/include/bli_pool_macro_defs.h rename to frame/base/bli_pool.h index c958595ae6..611c0feb0d 100644 --- a/frame/include/bli_pool_macro_defs.h +++ b/frame/base/bli_pool.h @@ -32,8 +32,63 @@ */ -#ifndef BLIS_POOL_MACRO_DEFS_H -#define BLIS_POOL_MACRO_DEFS_H +#ifndef BLIS_POOL_H +#define BLIS_POOL_H + +// -- Pool block type -- + +/* +typedef struct +{ + void* buf_sys; + void* buf_align; +} pblk_t; +*/ + +// -- Pool type -- + +/* +typedef struct +{ + pblk_t* block_ptrs; + dim_t block_ptrs_len; + + dim_t top_index; + dim_t num_blocks; + + siz_t block_size; + siz_t align_size; +} pool_t; +*/ + + +// Pool block query + +#define bli_pblk_buf_sys( pblk_p ) \ +\ + ( (pblk_p)->buf_sys ) + +#define bli_pblk_buf_align( pblk_p ) \ +\ + ( (pblk_p)->buf_align ) + +// Pool block modification + +#define bli_pblk_set_buf_sys( buf_sys0, pblk_p ) \ +{ \ + (pblk_p)->buf_sys = buf_sys0; \ +} + +#define bli_pblk_set_buf_align( buf_align0, pblk_p ) \ +{ \ + (pblk_p)->buf_align = buf_align0; \ +} + +#define bli_pblk_clear( pblk_p ) \ +{ \ + bli_pblk_set_buf_sys( NULL, pblk_p ); \ + bli_pblk_set_buf_align( NULL, pblk_p ); \ +} // Pool entry query @@ -42,6 +97,10 @@ \ ( (pool_p)->block_ptrs ) +#define bli_pool_block_ptrs_len( pool_p ) \ +\ + ( (pool_p)->block_ptrs_len ) + #define bli_pool_num_blocks( pool_p ) \ \ ( (pool_p)->num_blocks ) @@ -50,14 +109,18 @@ \ ( (pool_p)->block_size ) +#define bli_pool_align_size( pool_p ) \ +\ + ( (pool_p)->align_size ) + #define bli_pool_top_index( pool_p ) \ \ ( (pool_p)->top_index ) #define bli_pool_is_exhausted( pool_p ) \ \ - ( bli_pool_top_index( pool_p ) == -1 ) - + ( bli_pool_top_index( pool_p ) == \ + bli_pool_num_blocks( pool_p ) ) // Pool entry modification @@ -66,6 +129,11 @@ (pool_p)->block_ptrs = block_ptrs0; \ } +#define bli_pool_set_block_ptrs_len( block_ptrs_len0, pool_p ) \ +{ \ + (pool_p)->block_ptrs_len = block_ptrs_len0; \ +} + #define bli_pool_set_num_blocks( num_blocks0, pool_p ) \ { \ (pool_p)->num_blocks = num_blocks0; \ @@ -76,28 +144,37 @@ (pool_p)->block_size = block_size0; \ } -#define bli_pool_set_top_index( top_index0, pool_p ) \ +#define bli_pool_set_align_size( align_size0, pool_p ) \ { \ - (pool_p)->top_index = top_index0; \ + (pool_p)->align_size = align_size0; \ } -#define bli_pool_dec_top_index( pool_p ) \ +#define bli_pool_set_top_index( top_index0, pool_p ) \ { \ - ((pool_p)->top_index)--; \ + (pool_p)->top_index = top_index0; \ } -#define bli_pool_inc_top_index( pool_p ) \ -{ \ - ((pool_p)->top_index)++; \ -} +#endif -#define bli_pool_init( num_blocks, block_size, block_ptrs, pool_p ) \ -{ \ - bli_pool_set_num_blocks( num_blocks, pool_p ); \ - bli_pool_set_block_size( block_size, pool_p ); \ - bli_pool_set_block_ptrs( block_ptrs, pool_p ); \ - bli_pool_set_top_index( num_blocks - 1, pool_p ); \ -} +// ----------------------------------------------------------------------------- + +void bli_pool_init( dim_t num_blocks_init, + siz_t block_size, + siz_t align_size, + pool_t* pool ); +void bli_pool_finalize( pool_t* pool ); + +void bli_pool_checkout_block( pblk_t* block, pool_t* pool ); +void bli_pool_checkin_block( pblk_t* block, pool_t* pool ); + +void bli_pool_grow( dim_t num_blocks_add, pool_t* pool ); +void bli_pool_shrink( dim_t num_blocks_sub, pool_t* pool ); + +void bli_pool_alloc_block( siz_t block_size, + siz_t align_size, + pblk_t* block ); +void bli_pool_free_block( pblk_t* block ); +void bli_pool_print( pool_t* pool ); +void bli_pblk_print( pblk_t* pblk ); -#endif diff --git a/frame/include/bli_config_macro_defs.h b/frame/include/bli_config_macro_defs.h new file mode 100644 index 0000000000..3f33c7e66f --- /dev/null +++ b/frame/include/bli_config_macro_defs.h @@ -0,0 +1,178 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef BLIS_CONFIG_MACRO_DEFS_H +#define BLIS_CONFIG_MACRO_DEFS_H + + +// -- INTEGER PROPERTIES ------------------------------------------------------- + +// The bit size of the integer type used to track values such as dimensions, +// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed +// integers while 64 results in 64-bit integers. Any other value results in use +// of the C99 type "long int". Note that this ONLY affects integers used +// internally within BLIS as well as those exposed in the native BLAS-like BLIS +// interface. +#ifndef BLIS_INT_TYPE_SIZE +#define BLIS_INT_TYPE_SIZE 64 +#endif + + +// -- FLOATING-POINT PROPERTIES ------------------------------------------------ + +// Enable use of built-in C99 "float complex" and "double complex" types and +// associated overloaded operations and functions? Disabling results in +// scomplex and dcomplex being defined in terms of simple structs. +// NOTE: AVOID USING THIS FEATURE. IT IS PROBABLY BROKEN. +#ifdef BLIS_ENABLE_C99_COMPLEX + // No additional definitions needed. +#else + // Default behavior is disabled. +#endif + + +// -- MULTITHREADING ----------------------------------------------------------- + +// Enable multithreading via POSIX threads. +#ifdef BLIS_ENABLE_PTHREADS + // No additional definitions needed. +#else + // Default behavior is disabled. +#endif + +// Enable multithreading via OpenMP. +#ifdef BLIS_ENABLE_OPENMP + // No additional definitions needed. +#else + // Default behavior is disabled. +#endif + + +// -- MEMORY ALLOCATION -------------------------------------------------------- + +// Size of a virtual memory page. This is used to align certain memory +// buffers which are allocated and used internally. +#ifndef BLIS_PAGE_SIZE +#define BLIS_PAGE_SIZE 4096 +#endif + +// Alignment size (in bytes) needed by the instruction set for aligned +// SIMD/vector instructions. +#ifndef BLIS_SIMD_ALIGN_SIZE +#define BLIS_SIMD_ALIGN_SIZE 32 +#endif + +// Alignment size used to align local stack buffers within macro-kernel +// functions. +#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE + +// Alignment size used when allocating memory dynamically from the operating +// system (eg: posix_memalign()). To disable heap alignment and just use +// malloc() instead, set this to 1. +#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE + +// Alignment size used when sizing leading dimensions of dynamically +// allocated memory. +#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE + +// Alignment size used when allocating blocks to the internal memory +// pool (for packing buffers). +#define BLIS_POOL_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE + + +// -- MIXED DATATYPE SUPPORT --------------------------------------------------- + +// Basic (homogeneous) datatype support always enabled. + +// AVOID ENABLING MIXED DATATYPE SUPPORT! IT IS PROBABLY BROKEN. + +// Enable mixed domain operations? +//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT + +// Enable extra mixed precision operations? +//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT + + +// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- + +// Stay initialized after auto-initialization, unless and until the user +// explicitly calls bli_finalize(). +#ifdef BLIS_DISABLE_STAY_AUTO_INITIALIZED + #undef BLIS_ENABLE_STAY_AUTO_INITIALIZED +#else + // Default behavior is enabled. + #undef BLIS_ENABLE_STAY_AUTO_INITIALIZED // In case user explicitly enabled. + #define BLIS_ENABLE_STAY_AUTO_INITIALIZED +#endif + + +// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- + +// Enable the BLAS compatibility layer? +#ifdef BLIS_DISABLE_BLAS2BLIS + #undef BLIS_ENABLE_BLAS2BLIS +#else + // Default behavior is enabled. + #undef BLIS_ENABLE_BLAS2BLIS // In case user explicitly enabled. + #define BLIS_ENABLE_BLAS2BLIS +#endif + +// The bit size of the integer type used to track values such as dimensions and +// leading dimensions (ie: column strides) within the BLAS compatibility layer. +// A value of 32 results in the compatibility layer using 32-bit signed integers +// while 64 results in 64-bit integers. Any other value results in use of the +// C99 type "long int". Note that this ONLY affects integers used within the +// BLAS compatibility layer. +#ifndef BLIS_BLAS2BLIS_INT_TYPE_SIZE +#define BLIS_BLAS2BLIS_INT_TYPE_SIZE 64 +#endif + + +// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ + +// Enable the CBLAS compatibility layer? +// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer +// regardless of whether or not it was explicitly enabled above. Furthermore, +// the CBLAS compatibility layer will use the integer type size definition +// specified above when defining the size of its own integers (regardless of +// whether the BLAS layer was enabled directly or indirectly). +#ifdef BLIS_ENABLE_CBLAS + // No additional definitions needed. +#else + // Default behavior is disabled. +#endif + + +#endif + diff --git a/frame/include/bli_kernel_macro_defs.h b/frame/include/bli_kernel_macro_defs.h index e94b818c24..5622a098cf 100644 --- a/frame/include/bli_kernel_macro_defs.h +++ b/frame/include/bli_kernel_macro_defs.h @@ -1245,41 +1245,6 @@ #endif -// -- Define micro-panel alignment --------------------------------------------- - -// In this section, we consider each datatype-specific alignment sizes for -// micro-panels of A and B. If any definition is undefined, we define it to -// a safe default value (the size of the datatype). - -// Alignment for micro-panels of A -#ifndef BLIS_UPANEL_A_ALIGN_SIZE_S -#define BLIS_UPANEL_A_ALIGN_SIZE_S BLIS_SIZEOF_S -#endif -#ifndef BLIS_UPANEL_A_ALIGN_SIZE_D -#define BLIS_UPANEL_A_ALIGN_SIZE_D BLIS_SIZEOF_D -#endif -#ifndef BLIS_UPANEL_A_ALIGN_SIZE_C -#define BLIS_UPANEL_A_ALIGN_SIZE_C BLIS_SIZEOF_C -#endif -#ifndef BLIS_UPANEL_A_ALIGN_SIZE_Z -#define BLIS_UPANEL_A_ALIGN_SIZE_Z BLIS_SIZEOF_Z -#endif - -// Alignment for micro-panels of B -#ifndef BLIS_UPANEL_B_ALIGN_SIZE_S -#define BLIS_UPANEL_B_ALIGN_SIZE_S BLIS_SIZEOF_S -#endif -#ifndef BLIS_UPANEL_B_ALIGN_SIZE_D -#define BLIS_UPANEL_B_ALIGN_SIZE_D BLIS_SIZEOF_D -#endif -#ifndef BLIS_UPANEL_B_ALIGN_SIZE_C -#define BLIS_UPANEL_B_ALIGN_SIZE_C BLIS_SIZEOF_C -#endif -#ifndef BLIS_UPANEL_B_ALIGN_SIZE_Z -#define BLIS_UPANEL_B_ALIGN_SIZE_Z BLIS_SIZEOF_Z -#endif - - // -- Kernel blocksize checks -------------------------------------------------- // Verify that cache blocksizes are whole multiples of register blocksizes. diff --git a/frame/include/bli_macro_defs.h b/frame/include/bli_macro_defs.h index 4611df4f9c..907a07d835 100644 --- a/frame/include/bli_macro_defs.h +++ b/frame/include/bli_macro_defs.h @@ -92,6 +92,11 @@ #define MKSTR(s1) #s1 #define STRINGIFY_INT( s ) MKSTR( s ) +// Fortran-77 name-mangling macros. +#define PASTEF770(name) name ## _ +#define PASTEF77(ch1,name) ch1 ## name ## _ +#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ +#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ // -- Include other groups of macros @@ -100,7 +105,6 @@ #include "bli_gentprot_macro_defs.h" #include "bli_mem_macro_defs.h" -#include "bli_pool_macro_defs.h" #include "bli_obj_macro_defs.h" #include "bli_param_macro_defs.h" #include "bli_complex_macro_defs.h" diff --git a/frame/include/bli_mem_macro_defs.h b/frame/include/bli_mem_macro_defs.h index b603fdc634..51840b7126 100644 --- a/frame/include/bli_mem_macro_defs.h +++ b/frame/include/bli_mem_macro_defs.h @@ -38,17 +38,25 @@ // Mem entry query +#define bli_mem_pblk( mem_p ) \ +\ + ( &((mem_p)->pblk) ) + #define bli_mem_buffer( mem_p ) \ \ - ( (mem_p)->buf ) + ( bli_pblk_buf_align( bli_mem_pblk( mem_p ) ) ) + +#define bli_mem_buf_sys( mem_p ) \ +\ + ( bli_pblk_buf_sys( bli_mem_pblk( mem_p ) ) ) #define bli_mem_buf_type( mem_p ) \ \ - ( (mem_p)->buf_type ) + ( (mem_p)->buf_type ) #define bli_mem_pool( mem_p ) \ \ - ( (mem_p)->pool ) + ( (mem_p)->pool ) #define bli_mem_size( mem_p ) \ \ @@ -65,24 +73,42 @@ // Mem entry modification +#define bli_mem_set_pblk( pblk_p, mem_p ) \ +{ \ + mem_p->pblk = *(pblk_p); \ +} + #define bli_mem_set_buffer( buf0, mem_p ) \ { \ - mem_p->buf = buf0; \ + bli_pblk_set_buf_align( buf0, &(mem_p->pblk) ); \ +} + +#define bli_mem_set_buf_sys( buf0, mem_p ) \ +{ \ + bli_pblk_set_buf_sys( buf0, &(mem_p->pblk) ); \ } #define bli_mem_set_buf_type( buf_type0, mem_p ) \ { \ - mem_p->buf_type = buf_type0; \ + mem_p->buf_type = buf_type0; \ } #define bli_mem_set_pool( pool0, mem_p ) \ { \ - mem_p->pool = pool0; \ + mem_p->pool = pool0; \ } #define bli_mem_set_size( size0, mem_p ) \ { \ - mem_p->size = size0; \ + mem_p->size = size0; \ +} + +#define bli_mem_clear( mem_p ) \ +{ \ + bli_mem_set_buffer( NULL, mem_p ); \ + bli_mem_set_buf_sys( NULL, mem_p ); \ + bli_mem_set_pool( NULL, mem_p ); \ + bli_mem_set_size( 0, mem_p ); \ } diff --git a/frame/include/bli_mem_pool_macro_defs.h b/frame/include/bli_mem_pool_macro_defs.h deleted file mode 100644 index 6b3a6bc811..0000000000 --- a/frame/include/bli_mem_pool_macro_defs.h +++ /dev/null @@ -1,535 +0,0 @@ -/* - - BLIS - An object-based framework for developing high-performance BLAS-like - libraries. - - Copyright (C) 2014, The University of Texas at Austin - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - Neither the name of The University of Texas at Austin nor the names - of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -#ifndef BLIS_POOL_BLOCKS_MACRO_DEFS_H -#define BLIS_POOL_BLOCKS_MACRO_DEFS_H - - -// -- Memory pool block sizing macros ------------------------------------------ - -// In this file, we compute the memory pool block sizes for A, B, and C for -// each floating-point datatype, and then search for and save the maximum. -// The reason we settle on the largest is to prevent a developer from -// implementing a micro-kernel for one datatype (say, single real) and then -// end up in a situation where the memory pool is not large enough because -// the cache blocksize value of the datatype used to size the pool (e.g. -// double) was not set accordingly. - -// First we compute possible scaling factors for each datatype. These -// scaling factors actually take the form of numerator and denominator -// since we want stay in integer arithmetic. The purpose of the scaling -// factors is to increase the amount of space we reserve for the memory -// pool blocks if one of the packed micro-panels has a "leading dimension" -// that is larger than the register blocksize. (In this case, the leading -// dimension of a micro-panel is the packing register blocksize.) - -// Note that when computing the scaling factor, we have to determine which -// of PACKDIM_MR/DEFAULT_MR and PACKDIM_NR/DEFAULT_NR is greater so that -// the pair of values can be used to scale MAXIMUM_MC and MAXIMUM_NC. This -// is needed ONLY because the amount of space allocated for a block of A -// and a panel of B needs to be such that MR and NR can be swapped (ie: A -// is packed with NR and B is packed with MR). This transformation is -// needed for right-side trsm when inducing an algorithm that (a) has -// favorable access patterns for column-stored C and (b) allows the -// macro-kernel to reuse the existing left-side fused gemmtrsm micro-kernels. -// We cross-multiply so that the comparison can stay in integer arithmetic. - - -// -// Find the larger register blocksize for each datatype. -// -#if BLIS_DEFAULT_MR_S > BLIS_DEFAULT_NR_S -#define BLIS_MAX_MNR_S BLIS_DEFAULT_MR_S -#else -#define BLIS_MAX_MNR_S BLIS_DEFAULT_NR_S -#endif -#if BLIS_DEFAULT_MR_D > BLIS_DEFAULT_NR_D -#define BLIS_MAX_MNR_D BLIS_DEFAULT_MR_D -#else -#define BLIS_MAX_MNR_D BLIS_DEFAULT_NR_D -#endif -#if BLIS_DEFAULT_MR_C > BLIS_DEFAULT_NR_C -#define BLIS_MAX_MNR_C BLIS_DEFAULT_MR_C -#else -#define BLIS_MAX_MNR_C BLIS_DEFAULT_NR_C -#endif -#if BLIS_DEFAULT_MR_Z > BLIS_DEFAULT_NR_Z -#define BLIS_MAX_MNR_Z BLIS_DEFAULT_MR_Z -#else -#define BLIS_MAX_MNR_Z BLIS_DEFAULT_NR_Z -#endif - -// -// Define local maximum cache blocksizes -// - -// NOTE: We define these values here just to more concisely capture the -// increasing of the kc dimension blocksizes by the maximum register -// blocksize, which we do to make room for the nudging up of kc at -// runtime to be a multiple of MR or NR for triangular operations trmm, -// trmm3, and trsm. Also, we divide the induced values by 2 since they are -// defined in terms of real elements, but used (later, when computing -// pool block sizes) in terms of complex elements. - -#define BLIS_MAXIMUM_ASM_MC_S (BLIS_MAXIMUM_MC_S) -#define BLIS_MAXIMUM_ASM_KC_S ((BLIS_MAXIMUM_KC_S + BLIS_MAX_MNR_S)/2) -#define BLIS_MAXIMUM_ASM_NC_S (BLIS_MAXIMUM_NC_S) - -#define BLIS_MAXIMUM_ASM_MC_D (BLIS_MAXIMUM_MC_D) -#define BLIS_MAXIMUM_ASM_KC_D ((BLIS_MAXIMUM_KC_D + BLIS_MAX_MNR_D)/2) -#define BLIS_MAXIMUM_ASM_NC_D (BLIS_MAXIMUM_NC_D) - -#define BLIS_MAXIMUM_ASM_MC_C (BLIS_MAXIMUM_MC_C) -#define BLIS_MAXIMUM_ASM_KC_C ((BLIS_MAXIMUM_KC_C + BLIS_MAX_MNR_C)/2) -#define BLIS_MAXIMUM_ASM_NC_C (BLIS_MAXIMUM_NC_C) - -#define BLIS_MAXIMUM_ASM_MC_Z (BLIS_MAXIMUM_MC_Z) -#define BLIS_MAXIMUM_ASM_KC_Z ((BLIS_MAXIMUM_KC_Z + BLIS_MAX_MNR_Z)/2) -#define BLIS_MAXIMUM_ASM_NC_Z (BLIS_MAXIMUM_NC_Z) - -#define BLIS_MAXIMUM_IND_MC_C (BLIS_MAXIMUM_MC_S) -#define BLIS_MAXIMUM_IND_KC_C ((BLIS_MAXIMUM_KC_S + BLIS_MAX_MNR_S)/2) -#define BLIS_MAXIMUM_IND_NC_C (BLIS_MAXIMUM_NC_S) - -#define BLIS_MAXIMUM_IND_MC_Z (BLIS_MAXIMUM_MC_D) -#define BLIS_MAXIMUM_IND_KC_Z ((BLIS_MAXIMUM_KC_D + BLIS_MAX_MNR_D)/2) -#define BLIS_MAXIMUM_IND_NC_Z (BLIS_MAXIMUM_NC_D) - - -// -// Compute scaling factors for single real. -// -#if ( BLIS_PACKDIM_MR_S * BLIS_DEFAULT_NR_S ) >= \ - ( BLIS_PACKDIM_NR_S * BLIS_DEFAULT_MR_S ) - #define BLIS_PACKDIM_MAXR_S BLIS_PACKDIM_MR_S - #define BLIS_DEFAULT_MAXR_S BLIS_DEFAULT_MR_S -#else - #define BLIS_PACKDIM_MAXR_S BLIS_PACKDIM_NR_S - #define BLIS_DEFAULT_MAXR_S BLIS_DEFAULT_NR_S -#endif - -// -// Compute scaling factors for double real. -// -#if ( BLIS_PACKDIM_MR_D * BLIS_DEFAULT_NR_D ) >= \ - ( BLIS_PACKDIM_NR_D * BLIS_DEFAULT_MR_D ) - #define BLIS_PACKDIM_MAXR_D BLIS_PACKDIM_MR_D - #define BLIS_DEFAULT_MAXR_D BLIS_DEFAULT_MR_D -#else - #define BLIS_PACKDIM_MAXR_D BLIS_PACKDIM_NR_D - #define BLIS_DEFAULT_MAXR_D BLIS_DEFAULT_NR_D -#endif - -// -// Compute scaling factors for single complex. -// -#if ( BLIS_PACKDIM_MR_C * BLIS_DEFAULT_NR_C ) >= \ - ( BLIS_PACKDIM_NR_C * BLIS_DEFAULT_MR_C ) - #define BLIS_PACKDIM_MAXR_C BLIS_PACKDIM_MR_C - #define BLIS_DEFAULT_MAXR_C BLIS_DEFAULT_MR_C -#else - #define BLIS_PACKDIM_MAXR_C BLIS_PACKDIM_NR_C - #define BLIS_DEFAULT_MAXR_C BLIS_DEFAULT_NR_C -#endif - -// -// Compute scaling factors for double complex. -// -#if ( BLIS_PACKDIM_MR_Z * BLIS_DEFAULT_NR_Z ) >= \ - ( BLIS_PACKDIM_NR_Z * BLIS_DEFAULT_MR_Z ) - #define BLIS_PACKDIM_MAXR_Z BLIS_PACKDIM_MR_Z - #define BLIS_DEFAULT_MAXR_Z BLIS_DEFAULT_MR_Z -#else - #define BLIS_PACKDIM_MAXR_Z BLIS_PACKDIM_NR_Z - #define BLIS_DEFAULT_MAXR_Z BLIS_DEFAULT_NR_Z -#endif - - -// Next, we define the dimensions of the pool blocks for each datatype. - -// -// Compute pool dimensions for single real -// -#define BLIS_POOL_ASM_MC_S ( ( BLIS_MAXIMUM_ASM_MC_S * BLIS_PACKDIM_MAXR_S ) \ - / BLIS_DEFAULT_MAXR_S ) -#define BLIS_POOL_ASM_NC_S ( ( BLIS_MAXIMUM_ASM_NC_S * BLIS_PACKDIM_MAXR_S ) \ - / BLIS_DEFAULT_MAXR_S ) -#define BLIS_POOL_ASM_KC_S ( ( BLIS_MAXIMUM_ASM_KC_S * BLIS_PACKDIM_KR_S ) \ - / BLIS_DEFAULT_KR_S ) - -// -// Compute pool dimensions for double real -// -#define BLIS_POOL_ASM_MC_D ( ( BLIS_MAXIMUM_ASM_MC_D * BLIS_PACKDIM_MAXR_D ) \ - / BLIS_DEFAULT_MAXR_D ) -#define BLIS_POOL_ASM_NC_D ( ( BLIS_MAXIMUM_ASM_NC_D * BLIS_PACKDIM_MAXR_D ) \ - / BLIS_DEFAULT_MAXR_D ) -#define BLIS_POOL_ASM_KC_D ( ( BLIS_MAXIMUM_ASM_KC_D * BLIS_PACKDIM_KR_D ) \ - / BLIS_DEFAULT_KR_D ) - -// -// Compute pool dimensions for single complex (native) -// -#define BLIS_POOL_ASM_MC_C ( ( BLIS_MAXIMUM_ASM_MC_C * BLIS_PACKDIM_MAXR_C ) \ - / BLIS_DEFAULT_MAXR_C ) -#define BLIS_POOL_ASM_NC_C ( ( BLIS_MAXIMUM_ASM_NC_C * BLIS_PACKDIM_MAXR_C ) \ - / BLIS_DEFAULT_MAXR_C ) -#define BLIS_POOL_ASM_KC_C ( ( BLIS_MAXIMUM_ASM_KC_C * BLIS_PACKDIM_KR_C ) \ - / BLIS_DEFAULT_KR_C ) - -// -// Compute pool dimensions for double complex (native) -// -#define BLIS_POOL_ASM_MC_Z ( ( BLIS_MAXIMUM_ASM_MC_Z * BLIS_PACKDIM_MAXR_Z ) \ - / BLIS_DEFAULT_MAXR_Z ) -#define BLIS_POOL_ASM_NC_Z ( ( BLIS_MAXIMUM_ASM_NC_Z * BLIS_PACKDIM_MAXR_Z ) \ - / BLIS_DEFAULT_MAXR_Z ) -#define BLIS_POOL_ASM_KC_Z ( ( BLIS_MAXIMUM_ASM_KC_Z * BLIS_PACKDIM_KR_Z ) \ - / BLIS_DEFAULT_KR_Z ) - -// -// Compute pool dimensions for single complex (induced) -// -#define BLIS_POOL_IND_MC_C ( ( BLIS_MAXIMUM_IND_MC_C * BLIS_PACKDIM_MAXR_S ) \ - / BLIS_DEFAULT_MAXR_S ) -#define BLIS_POOL_IND_NC_C ( ( BLIS_MAXIMUM_IND_NC_C * BLIS_PACKDIM_MAXR_S ) \ - / BLIS_DEFAULT_MAXR_S ) -#define BLIS_POOL_IND_KC_C ( ( BLIS_MAXIMUM_IND_KC_C * BLIS_PACKDIM_KR_S ) \ - / BLIS_DEFAULT_KR_S ) - -// -// Compute pool dimensions for double complex (induced) -// -#define BLIS_POOL_IND_MC_Z ( ( BLIS_MAXIMUM_IND_MC_Z * BLIS_PACKDIM_MAXR_D ) \ - / BLIS_DEFAULT_MAXR_D ) -#define BLIS_POOL_IND_NC_Z ( ( BLIS_MAXIMUM_IND_NC_Z * BLIS_PACKDIM_MAXR_D ) \ - / BLIS_DEFAULT_MAXR_D ) -#define BLIS_POOL_IND_KC_Z ( ( BLIS_MAXIMUM_IND_KC_Z * BLIS_PACKDIM_KR_D ) \ - / BLIS_DEFAULT_KR_D ) - - -// Now, we compute the size of each block/panel of A, B, and C for each -// datatype. - -// NOTE: We assume the worst case of unit register blocksizes, and -// therefore add a full micro-panel alignment value to KC. This can -// result in quite a bit of unused space, but it's better than the -// alternative of being bitten by the absolute black plague that -// would result from overflowing a block within the pool. - -// -// Compute memory pool block sizes for single real. -// - -#define BLIS_MK_BLOCK_SIZE_ASM_S ( BLIS_POOL_ASM_MC_S * \ - ( BLIS_POOL_ASM_KC_S + \ - ( BLIS_UPANEL_A_ALIGN_SIZE_S / \ - BLIS_SIZEOF_S ) \ - ) * \ - BLIS_SIZEOF_S \ - ) -#define BLIS_KN_BLOCK_SIZE_ASM_S ( \ - ( BLIS_POOL_ASM_KC_S + \ - ( BLIS_UPANEL_B_ALIGN_SIZE_S / \ - BLIS_SIZEOF_S ) \ - ) * \ - BLIS_POOL_ASM_NC_S * \ - BLIS_SIZEOF_S \ - ) -#define BLIS_MN_BLOCK_SIZE_ASM_S ( BLIS_POOL_ASM_MC_S * \ - BLIS_POOL_ASM_NC_S * \ - BLIS_SIZEOF_S \ - ) - -// -// Compute memory pool block sizes for double real. -// - -#define BLIS_MK_BLOCK_SIZE_ASM_D ( BLIS_POOL_ASM_MC_D * \ - ( BLIS_POOL_ASM_KC_D + \ - ( BLIS_UPANEL_A_ALIGN_SIZE_D / \ - BLIS_SIZEOF_D ) \ - ) * \ - BLIS_SIZEOF_D \ - ) -#define BLIS_KN_BLOCK_SIZE_ASM_D ( \ - ( BLIS_POOL_ASM_KC_D + \ - ( BLIS_UPANEL_B_ALIGN_SIZE_D / \ - BLIS_SIZEOF_D ) \ - ) * \ - BLIS_POOL_ASM_NC_D * \ - BLIS_SIZEOF_D \ - ) -#define BLIS_MN_BLOCK_SIZE_ASM_D ( BLIS_POOL_ASM_MC_D * \ - BLIS_POOL_ASM_NC_D * \ - BLIS_SIZEOF_D \ - ) - -// -// Compute memory pool block sizes for single complex. -// - -#define BLIS_MK_BLOCK_SIZE_ASM_C ( BLIS_POOL_ASM_MC_C * \ - ( BLIS_POOL_ASM_KC_C + \ - ( BLIS_UPANEL_A_ALIGN_SIZE_C / \ - BLIS_SIZEOF_C ) \ - ) * \ - BLIS_SIZEOF_C \ - ) -#define BLIS_KN_BLOCK_SIZE_ASM_C ( \ - ( BLIS_POOL_ASM_KC_C + \ - ( BLIS_UPANEL_B_ALIGN_SIZE_C / \ - BLIS_SIZEOF_C ) \ - ) * \ - BLIS_POOL_ASM_NC_C * \ - BLIS_SIZEOF_C \ - ) -#define BLIS_MN_BLOCK_SIZE_ASM_C ( BLIS_POOL_ASM_MC_C * \ - BLIS_POOL_ASM_NC_C * \ - BLIS_SIZEOF_C \ - ) - -// -// Compute memory pool block sizes for double complex. -// - -#define BLIS_MK_BLOCK_SIZE_ASM_Z ( BLIS_POOL_ASM_MC_Z * \ - ( BLIS_POOL_ASM_KC_Z + \ - ( BLIS_UPANEL_A_ALIGN_SIZE_Z / \ - BLIS_SIZEOF_Z ) \ - ) * \ - BLIS_SIZEOF_Z \ - ) -#define BLIS_KN_BLOCK_SIZE_ASM_Z ( \ - ( BLIS_POOL_ASM_KC_Z + \ - ( BLIS_UPANEL_B_ALIGN_SIZE_Z / \ - BLIS_SIZEOF_Z ) \ - ) * \ - BLIS_POOL_ASM_NC_Z * \ - BLIS_SIZEOF_Z \ - ) -#define BLIS_MN_BLOCK_SIZE_ASM_Z ( BLIS_POOL_ASM_MC_Z * \ - BLIS_POOL_ASM_NC_Z * \ - BLIS_SIZEOF_Z \ - ) - -// -// Compute memory pool block sizes for single complex (induced). -// - -// NOTE: We scale by 3/2 because 3m1 requires 50% more space than other -// algorithms. - -#define BLIS_MK_BLOCK_SIZE_IND_C ( BLIS_POOL_IND_MC_C * \ - ( BLIS_POOL_IND_KC_C + \ - ( BLIS_UPANEL_A_ALIGN_SIZE_C / \ - BLIS_SIZEOF_C ) \ - ) * \ - ( BLIS_SIZEOF_C * \ - 3 \ - ) / 2 \ - ) -#define BLIS_KN_BLOCK_SIZE_IND_C ( \ - ( BLIS_POOL_IND_KC_C + \ - ( BLIS_UPANEL_B_ALIGN_SIZE_C / \ - BLIS_SIZEOF_C ) \ - ) * \ - BLIS_POOL_IND_NC_C * \ - ( BLIS_SIZEOF_C * \ - 3 \ - ) / 2 \ - ) -#define BLIS_MN_BLOCK_SIZE_IND_C ( BLIS_POOL_IND_MC_C * \ - BLIS_POOL_IND_NC_C * \ - ( BLIS_SIZEOF_C * \ - 3 \ - ) / 2 \ - ) - -// -// Compute memory pool block sizes for double complex (induced). -// - -// NOTE: We scale by 3/2 because 3m1 requires 50% more space than other -// algorithms. - -#define BLIS_MK_BLOCK_SIZE_IND_Z ( BLIS_POOL_IND_MC_Z * \ - ( BLIS_POOL_IND_KC_Z + \ - ( BLIS_UPANEL_A_ALIGN_SIZE_Z / \ - BLIS_SIZEOF_Z ) \ - ) * \ - ( BLIS_SIZEOF_Z * \ - 3 \ - ) / 2 \ - ) -#define BLIS_KN_BLOCK_SIZE_IND_Z ( \ - ( BLIS_POOL_IND_KC_Z + \ - ( BLIS_UPANEL_B_ALIGN_SIZE_Z / \ - BLIS_SIZEOF_Z ) \ - ) * \ - BLIS_POOL_IND_NC_Z * \ - ( BLIS_SIZEOF_Z * \ - 3 \ - ) / 2 \ - ) -#define BLIS_MN_BLOCK_SIZE_IND_Z ( BLIS_POOL_IND_MC_Z * \ - BLIS_POOL_IND_NC_Z * \ - ( BLIS_SIZEOF_Z * \ - 3 \ - ) / 2 \ - ) - - -// -- Maximum block size search ------------------------------------------------ - -// In this section, we find the largest of each block size. - -// -// Find the largest block size for blocks of A. -// -#define BLIS_MK_BLOCK_SIZE BLIS_MK_BLOCK_SIZE_ASM_S -#if BLIS_MK_BLOCK_SIZE_ASM_D > BLIS_MK_BLOCK_SIZE -#undef BLIS_MK_BLOCK_SIZE -#define BLIS_MK_BLOCK_SIZE BLIS_MK_BLOCK_SIZE_ASM_D -#endif -#if BLIS_MK_BLOCK_SIZE_ASM_C > BLIS_MK_BLOCK_SIZE -#undef BLIS_MK_BLOCK_SIZE -#define BLIS_MK_BLOCK_SIZE BLIS_MK_BLOCK_SIZE_ASM_C -#endif -#if BLIS_MK_BLOCK_SIZE_ASM_Z > BLIS_MK_BLOCK_SIZE -#undef BLIS_MK_BLOCK_SIZE -#define BLIS_MK_BLOCK_SIZE BLIS_MK_BLOCK_SIZE_ASM_Z -#endif -#if BLIS_MK_BLOCK_SIZE_IND_C > BLIS_MK_BLOCK_SIZE -#undef BLIS_MK_BLOCK_SIZE -#define BLIS_MK_BLOCK_SIZE BLIS_MK_BLOCK_SIZE_IND_C -#endif -#if BLIS_MK_BLOCK_SIZE_IND_Z > BLIS_MK_BLOCK_SIZE -#undef BLIS_MK_BLOCK_SIZE -#define BLIS_MK_BLOCK_SIZE BLIS_MK_BLOCK_SIZE_IND_Z -#endif - -// -// Find the largest block size for panels of B. -// -#define BLIS_KN_BLOCK_SIZE BLIS_KN_BLOCK_SIZE_ASM_S -#if BLIS_KN_BLOCK_SIZE_ASM_D > BLIS_KN_BLOCK_SIZE -#undef BLIS_KN_BLOCK_SIZE -#define BLIS_KN_BLOCK_SIZE BLIS_KN_BLOCK_SIZE_ASM_D -#endif -#if BLIS_KN_BLOCK_SIZE_ASM_C > BLIS_KN_BLOCK_SIZE -#undef BLIS_KN_BLOCK_SIZE -#define BLIS_KN_BLOCK_SIZE BLIS_KN_BLOCK_SIZE_ASM_C -#endif -#if BLIS_KN_BLOCK_SIZE_ASM_Z > BLIS_KN_BLOCK_SIZE -#undef BLIS_KN_BLOCK_SIZE -#define BLIS_KN_BLOCK_SIZE BLIS_KN_BLOCK_SIZE_ASM_Z -#endif -#if BLIS_KN_BLOCK_SIZE_IND_C > BLIS_KN_BLOCK_SIZE -#undef BLIS_KN_BLOCK_SIZE -#define BLIS_KN_BLOCK_SIZE BLIS_KN_BLOCK_SIZE_IND_C -#endif -#if BLIS_KN_BLOCK_SIZE_IND_Z > BLIS_KN_BLOCK_SIZE -#undef BLIS_KN_BLOCK_SIZE -#define BLIS_KN_BLOCK_SIZE BLIS_KN_BLOCK_SIZE_IND_Z -#endif - -// -// Find the largest block size for panels of C. -// -#define BLIS_MN_BLOCK_SIZE BLIS_MN_BLOCK_SIZE_ASM_S -#if BLIS_MN_BLOCK_SIZE_ASM_D > BLIS_MN_BLOCK_SIZE -#undef BLIS_MN_BLOCK_SIZE -#define BLIS_MN_BLOCK_SIZE BLIS_MN_BLOCK_SIZE_ASM_D -#endif -#if BLIS_MN_BLOCK_SIZE_ASM_C > BLIS_MN_BLOCK_SIZE -#undef BLIS_MN_BLOCK_SIZE -#define BLIS_MN_BLOCK_SIZE BLIS_MN_BLOCK_SIZE_ASM_C -#endif -#if BLIS_MN_BLOCK_SIZE_ASM_Z > BLIS_MN_BLOCK_SIZE -#undef BLIS_MN_BLOCK_SIZE -#define BLIS_MN_BLOCK_SIZE BLIS_MN_BLOCK_SIZE_ASM_Z -#endif -#if BLIS_MN_BLOCK_SIZE_IND_C > BLIS_MN_BLOCK_SIZE -#undef BLIS_MN_BLOCK_SIZE -#define BLIS_MN_BLOCK_SIZE BLIS_MN_BLOCK_SIZE_IND_C -#endif -#if BLIS_MN_BLOCK_SIZE_IND_Z > BLIS_MN_BLOCK_SIZE -#undef BLIS_MN_BLOCK_SIZE -#define BLIS_MN_BLOCK_SIZE BLIS_MN_BLOCK_SIZE_IND_Z -#endif - - -// -- Compute pool sizes ------------------------------------------------------- - - -// Define each pool's total size using the block sizes determined above. -// These values are used in bli_mem.c to size the static memory pool -// arrays. - -// -// Pool for MC x KC blocks of A. -// -#define BLIS_MK_POOL_SIZE ( \ - BLIS_NUM_MC_X_KC_BLOCKS * \ - ( BLIS_MK_BLOCK_SIZE + \ - BLIS_CONTIG_ADDR_ALIGN_SIZE \ - ) + \ - BLIS_MAX_PRELOAD_BYTE_OFFSET \ - ) - -// -// Pool for KC x NC panels of B. -// -#define BLIS_KN_POOL_SIZE ( \ - BLIS_NUM_KC_X_NC_BLOCKS * \ - ( BLIS_KN_BLOCK_SIZE + \ - BLIS_CONTIG_ADDR_ALIGN_SIZE \ - ) + \ - BLIS_MAX_PRELOAD_BYTE_OFFSET \ - ) - -// -// Pool for MC x NC panels of C. -// -#define BLIS_MN_POOL_SIZE ( \ - BLIS_NUM_MC_X_NC_BLOCKS * \ - ( BLIS_MN_BLOCK_SIZE + \ - BLIS_CONTIG_ADDR_ALIGN_SIZE \ - ) + \ - BLIS_MAX_PRELOAD_BYTE_OFFSET \ - ) - - -#endif diff --git a/frame/include/bli_type_defs.h b/frame/include/bli_type_defs.h index e5d83bef3b..a8f5bcb5b1 100644 --- a/frame/include/bli_type_defs.h +++ b/frame/include/bli_type_defs.h @@ -91,6 +91,11 @@ typedef guint_t objbits_t; // object information bit field // -- Real types -- +// Define the number of floating-point types supported, and the size of the +// largest type. +#define BLIS_NUM_FP_TYPES 4 +#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) + // There are some places where we need to use sizeof() inside of a C // preprocessor #if conditional, and so here we define the various sizes // for those purposes. @@ -487,26 +492,41 @@ typedef enum // -- BLIS misc. structure types ----------------------------------------------- // -// -- Memory pool type -- +// -- Pool block type -- + +typedef struct +{ + void* buf_sys; + void* buf_align; +} pblk_t; + +// -- Pool type -- typedef struct { - void** block_ptrs; - gint_t top_index; - siz_t num_blocks; - siz_t block_size; + pblk_t* block_ptrs; + dim_t block_ptrs_len; + + dim_t top_index; + dim_t num_blocks; + + siz_t block_size; + siz_t align_size; } pool_t; // -- Memory object type -- typedef struct mem_s { - void* buf; + pblk_t pblk; packbuf_t buf_type; pool_t* pool; siz_t size; } mem_t; +// -- Memory block type -- + + // -- Blocksize object type -- typedef struct blksz_s @@ -711,17 +731,17 @@ typedef enum typedef enum { - BLIS_MACH_EPS = 0, - BLIS_MACH_SFMIN, - BLIS_MACH_BASE, - BLIS_MACH_PREC, - BLIS_MACH_NDIGMANT, - BLIS_MACH_RND, - BLIS_MACH_EMIN, - BLIS_MACH_RMIN, - BLIS_MACH_EMAX, - BLIS_MACH_RMAX, - BLIS_MACH_EPS2 + BLIS_MACH_EPS = 0, + BLIS_MACH_SFMIN, + BLIS_MACH_BASE, + BLIS_MACH_PREC, + BLIS_MACH_NDIGMANT, + BLIS_MACH_RND, + BLIS_MACH_EMIN, + BLIS_MACH_RMIN, + BLIS_MACH_EMAX, + BLIS_MACH_RMAX, + BLIS_MACH_EPS2 } machval_t; #define BLIS_NUM_MACH_PARAMS 11 diff --git a/frame/include/blis.h b/frame/include/blis.h index 045cdc894d..2b27908023 100644 --- a/frame/include/blis.h +++ b/frame/include/blis.h @@ -54,6 +54,7 @@ extern "C" { // KNOW WHAT YOU ARE DOING. #include "bli_config.h" +#include "bli_config_macro_defs.h" // -- System headers -- @@ -98,7 +99,7 @@ extern "C" { // -- BLIS memory pool definitions -- -#include "bli_mem_pool_macro_defs.h" +//#include "bli_mem_pool_macro_defs.h" // -- Base operation prototypes -- @@ -108,6 +109,8 @@ extern "C" { #include "bli_malloc.h" #include "bli_obj.h" #include "bli_obj_scalar.h" +#include "bli_ind.h" +#include "bli_pool.h" #include "bli_mem.h" #include "bli_part.h" #include "bli_query.h" @@ -123,7 +126,6 @@ extern "C" { #include "bli_opid.h" #include "bli_flops.h" #include "bli_cntl.h" -#include "bli_ind.h" #include "bli_info.h" diff --git a/frame/ind/query/bli_bsv_query.c b/frame/ind/query/bli_bsv_query.c index 5124a19939..73f4a7738f 100644 --- a/frame/ind/query/bli_bsv_query.c +++ b/frame/ind/query/bli_bsv_query.c @@ -167,11 +167,11 @@ blksz_t* bli_bsv_get_avail_blksz( bszid_t bsv, opid_t oper, num_t dt ) blksz_t* bli_bsv_get_blksz( bszid_t bsv, ind_t method ) { - // Initialize BLIS, if it isn't already initialized. This is - // needed because we have to ensure that the blksz_t objects - // have been created, otherwise this function could return a - // NULL (or garbage) address. - bli_init(); + // Initialize the cntl API, if it isn't already initialized. This is + // needed because we have to ensure that the blksz_t objects have + // been created, otherwise this function could return a NULL (or + // garbage) address. + bli_cntl_init(); return *(bli_bsizes[ method ][ bsv ]); } diff --git a/frame/ind/query/bli_ind_query.c b/frame/ind/query/bli_ind_query.c index 1e0f563978..d07abc7aaf 100644 --- a/frame/ind/query/bli_ind_query.c +++ b/frame/ind/query/bli_ind_query.c @@ -34,8 +34,6 @@ #include "blis.h" -static bool_t bli_ind_is_init = FALSE; - static void* bli_ind_oper_fp[BLIS_NUM_IND_METHODS][BLIS_NUM_LEVEL3_OPS] = { /* gemm hemm herk her2k symm syrk, syr2k trmm3 trmm trsm */ @@ -213,6 +211,8 @@ char* bli_ind_oper_get_avail_impl_string( opid_t oper, num_t dt ) // ----------------------------------------------------------------------------- +static bool_t bli_ind_is_init = FALSE; + void bli_ind_init( void ) { // If the API is already initialized, return early. diff --git a/frame/ind/query/bli_ukr_query.c b/frame/ind/query/bli_ukr_query.c index 464016300e..61d07d6fc0 100644 --- a/frame/ind/query/bli_ukr_query.c +++ b/frame/ind/query/bli_ukr_query.c @@ -200,10 +200,10 @@ func_t* bli_ukr_get_funcs( l3ukr_t ukr, ind_t method ) { func_t** p = bli_ukrs[ method ][ ukr ]; - // Initialize BLIS, if it isn't already initialized. This is - // needed because we have to ensure that the ukr func_t objects - // have been created (and thus contain valid function pointers). - bli_init(); + // Initialize the cntl API, if it isn't already initialized. This is + // needed because we have to ensure that the ukr func_t objects have + // been created (and thus contain valid function pointers). + bli_cntl_init(); // Avoid dereferencing NULL pointers. (A NULL pointer indicates that // there is no kernel for the requested kernel type and method.) @@ -215,10 +215,10 @@ func_t* bli_ukr_get_ref_funcs( l3ukr_t ukr ) { func_t** p = bli_ref_ukrs[ ukr ]; - // Initialize BLIS, if it isn't already initialized. This is - // needed because we have to ensure that the ukr func_t objects - // have been created (and thus contain valid function pointers). - bli_init(); + // Initialize the cntl API, if it isn't already initialized. This is + // needed because we have to ensure that the ukr func_t objects have + // been created (and thus contain valid function pointers). + bli_cntl_init(); // Avoid dereferencing NULL pointers. (A NULL pointer indicates that // there is no reference kernel for the requested kernel type.) diff --git a/testsuite/src/test_addv.c b/testsuite/src/test_addv.c index b8909944a3..8f2cf880ec 100644 --- a/testsuite/src/test_addv.c +++ b/testsuite/src/test_addv.c @@ -164,7 +164,7 @@ void libblis_test_addv_experiment( test_params_t* params, } // Estimate the performance of the best experiment repeat. - *perf = ( 2.0 * m ) / time_min / FLOPS_PER_UNIT_PERF; + *perf = ( 1.0 * m ) / time_min / FLOPS_PER_UNIT_PERF; if ( bli_obj_is_complex( x ) ) *perf *= 2.0; // Perform checks. diff --git a/testsuite/src/test_libblis.c b/testsuite/src/test_libblis.c index a493a3a853..e6ae326480 100644 --- a/testsuite/src/test_libblis.c +++ b/testsuite/src/test_libblis.c @@ -595,39 +595,27 @@ void libblis_test_output_params_struct( FILE* os, test_params_t* params ) libblis_test_fprintf_c( os, "\n" ); libblis_test_fprintf_c( os, "--- BLIS library info -------------------------------------\n" ); libblis_test_fprintf_c( os, "\n" ); - libblis_test_fprintf_c( os, "version string %s\n", bli_info_get_version_str() ); + libblis_test_fprintf_c( os, "version string %s\n", bli_info_get_version_str() ); libblis_test_fprintf_c( os, "\n" ); - libblis_test_fprintf_c( os, "--- BLIS config header ---\n" ); + libblis_test_fprintf_c( os, "--- BLIS configuration info ---\n" ); libblis_test_fprintf_c( os, "\n" ); - libblis_test_fprintf_c( os, "integer type size (bits) %d\n", ( int )int_type_size ); - libblis_test_fprintf_c( os, "# of floating-point types %d\n", ( int )bli_info_get_num_fp_types() ); - libblis_test_fprintf_c( os, "maximum type size %d\n", ( int )bli_info_get_max_type_size() ); + libblis_test_fprintf_c( os, "integer type size (bits) %d\n", ( int )int_type_size ); libblis_test_fprintf_c( os, "\n" ); - libblis_test_fprintf_c( os, "maximum number of threads %d\n", ( int )bli_info_get_max_num_threads() ); + libblis_test_fprintf_c( os, "SIMD alignment (bytes) %d\n", ( int )bli_info_get_simd_align_size() ); + libblis_test_fprintf_c( os, "Page size (bytes) %d\n", ( int )bli_info_get_page_size() ); libblis_test_fprintf_c( os, "\n" ); - libblis_test_fprintf_c( os, "SIMD alignment (bytes) %d\n", ( int )bli_info_get_simd_align_size() ); + libblis_test_fprintf_c( os, "memory alignment (bytes) \n" ); + libblis_test_fprintf_c( os, " stack address (def: simd) %d\n", ( int )bli_info_get_stack_buf_align_size() ); + libblis_test_fprintf_c( os, " obj_t address (def: simd) %d\n", ( int )bli_info_get_heap_addr_align_size() ); + libblis_test_fprintf_c( os, " obj_t stride (def: simd) %d\n", ( int )bli_info_get_heap_stride_align_size() ); + libblis_test_fprintf_c( os, " pool block addr (def: page) %d\n", ( int )bli_info_get_pool_addr_align_size() ); libblis_test_fprintf_c( os, "\n" ); - libblis_test_fprintf_c( os, "stack memory allocation \n" ); - libblis_test_fprintf_c( os, " address alignment (bytes) %d\n", ( int )bli_info_get_stack_buf_align_size() ); + libblis_test_fprintf_c( os, "BLAS compatibility layer \n" ); + libblis_test_fprintf_c( os, " enabled? %d\n", ( int )bli_info_get_enable_blas2blis() ); + libblis_test_fprintf_c( os, " integer type size (bits) %d\n", ( int )bli_info_get_blas2blis_int_type_size() ); libblis_test_fprintf_c( os, "\n" ); - libblis_test_fprintf_c( os, "dynamic memory allocation \n" ); - libblis_test_fprintf_c( os, " address alignment %d\n", ( int )bli_info_get_heap_addr_align_size() ); - libblis_test_fprintf_c( os, " stride alignment %d\n", ( int )bli_info_get_heap_stride_align_size() ); - libblis_test_fprintf_c( os, "\n" ); - libblis_test_fprintf_c( os, "contiguous memory allocation \n" ); - libblis_test_fprintf_c( os, " # of mc x kc blocks %d\n", ( int )bli_info_get_num_mc_x_kc_blocks() ); - libblis_test_fprintf_c( os, " # of kc x nc blocks %d\n", ( int )bli_info_get_num_kc_x_nc_blocks() ); - libblis_test_fprintf_c( os, " # of mc x nc blocks %d\n", ( int )bli_info_get_num_mc_x_nc_blocks() ); - libblis_test_fprintf_c( os, " block address alignment %d\n", ( int )bli_info_get_contig_addr_align_size() ); - libblis_test_fprintf_c( os, " max preload byte offset %d\n", ( int )bli_info_get_max_preload_byte_offset() ); - libblis_test_fprintf_c( os, " actual pool sizes (bytes) \n" ); - libblis_test_fprintf_c( os, " for mc x kc blocks of A %d\n", ( int )bli_info_get_mk_pool_size() ); - libblis_test_fprintf_c( os, " for kc x nc panels of B %d\n", ( int )bli_info_get_kn_pool_size() ); - libblis_test_fprintf_c( os, " for mc x nc panels of C %d\n", ( int )bli_info_get_mn_pool_size() ); - libblis_test_fprintf_c( os, "\n" ); - libblis_test_fprintf_c( os, "BLAS compatibility layer \n" ); - libblis_test_fprintf_c( os, " enabled? %d\n", ( int )bli_info_get_enable_blas2blis() ); - libblis_test_fprintf_c( os, " integer type size (bits) %d\n", ( int )bli_info_get_blas2blis_int_type_size() ); + libblis_test_fprintf_c( os, "CBLAS compatibility layer \n" ); + libblis_test_fprintf_c( os, " enabled? %d\n", ( int )bli_info_get_enable_cblas() ); libblis_test_fprintf_c( os, "\n" ); libblis_test_fprintf_c( os, "floating-point types s d c z \n" ); libblis_test_fprintf_c( os, " sizes (bytes) %7u %7u %7u %7u\n", sizeof(float), @@ -862,18 +850,6 @@ void libblis_test_output_params_struct( FILE* os, test_params_t* params ) libblis_test_fprintf_c( os, "\n" ); libblis_test_fprintf_c( os, "--- BLIS misc. other info ---\n" ); libblis_test_fprintf_c( os, "\n" ); - libblis_test_fprintf_c( os, "micro-panel alignment (bytes) s d c z \n" ); - libblis_test_fprintf_c( os, " A (left matrix) %7d %7d %7d %7d\n", - ( int )bli_info_get_upanel_a_align_size_s(), - ( int )bli_info_get_upanel_a_align_size_d(), - ( int )bli_info_get_upanel_a_align_size_c(), - ( int )bli_info_get_upanel_a_align_size_z() ); - libblis_test_fprintf_c( os, " B (right matrix) %7d %7d %7d %7d\n", - ( int )bli_info_get_upanel_b_align_size_s(), - ( int )bli_info_get_upanel_b_align_size_d(), - ( int )bli_info_get_upanel_b_align_size_c(), - ( int )bli_info_get_upanel_b_align_size_z() ); - libblis_test_fprintf_c( os, "\n" ); libblis_test_fprintf_c( os, "level-2 cache blocksizes s d c z \n" ); libblis_test_fprintf_c( os, " m dimension %7d %7d %7d %7d\n", ( int )bli_info_get_default_l2_mc_s(), diff --git a/testsuite/src/test_scal2m.c b/testsuite/src/test_scal2m.c index 2816c8a08e..8cd1fdd2bc 100644 --- a/testsuite/src/test_scal2m.c +++ b/testsuite/src/test_scal2m.c @@ -181,7 +181,7 @@ void libblis_test_scal2m_experiment( test_params_t* params, } // Estimate the performance of the best experiment repeat. - *perf = ( 2.0 * m * n ) / time_min / FLOPS_PER_UNIT_PERF; + *perf = ( 1.0 * m * n ) / time_min / FLOPS_PER_UNIT_PERF; if ( bli_obj_is_complex( y ) ) *perf *= 4.0; // Perform checks. diff --git a/testsuite/src/test_scal2v.c b/testsuite/src/test_scal2v.c index 184b50067a..803e5e2cf8 100644 --- a/testsuite/src/test_scal2v.c +++ b/testsuite/src/test_scal2v.c @@ -179,7 +179,7 @@ void libblis_test_scal2v_experiment( test_params_t* params, } // Estimate the performance of the best experiment repeat. - *perf = ( 2.0 * m ) / time_min / FLOPS_PER_UNIT_PERF; + *perf = ( 1.0 * m ) / time_min / FLOPS_PER_UNIT_PERF; if ( bli_obj_is_complex( y ) ) *perf *= 4.0; // Perform checks. diff --git a/testsuite/src/test_subv.c b/testsuite/src/test_subv.c index 2b7f8c3cea..c6379bf1eb 100644 --- a/testsuite/src/test_subv.c +++ b/testsuite/src/test_subv.c @@ -165,7 +165,7 @@ void libblis_test_subv_experiment( test_params_t* params, } // Estimate the performance of the best experiment repeat. - *perf = ( 2.0 * m ) / time_min / FLOPS_PER_UNIT_PERF; + *perf = ( 1.0 * m ) / time_min / FLOPS_PER_UNIT_PERF; if ( bli_obj_is_complex( x ) ) *perf *= 2.0; // Perform checks. From 267253de8a7be546ce87626443ee38701c1d411f Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Fri, 19 Jun 2015 12:01:49 -0500 Subject: [PATCH 10/38] Version file update (0.1.7) --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index c946ee6160..11808190d4 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.1.6 +0.1.7 From 0b7255a642d56723f02d7ca1f8f21809967b8515 Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Fri, 19 Jun 2015 12:01:50 -0500 Subject: [PATCH 11/38] CHANGELOG update (0.1.7) --- CHANGELOG | 851 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 833 insertions(+), 18 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index ee843471e0..e579f7f4a4 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,10 +1,825 @@ -commit 38ea5022e4ed846112198c4e1672fcdaeb90dc71 (HEAD, 0.1.6, master) +commit 267253de8a7be546ce87626443ee38701c1d411f (HEAD -> master, tag: 0.1.7) +Author: Field G. Van Zee +Date: Fri Jun 19 12:01:49 2015 -0500 + + Version file update (0.1.7) + +commit 7cd01b71b5e757a6774625b3c9f427f5e7664a76 (origin/master, origin/HEAD) +Author: Field G. Van Zee +Date: Fri Jun 19 11:31:53 2015 -0500 + + Implemented dynamic allocation for packing buffers. + + Details: + - Replaced the old memory allocator, which was based on statically- + allocated arrays, with one based on a new internal pool_t type, which, + combined with a new bli_pool_*() API, provides a new abstract data + type that implements the same memory pool functionality but with blocks + from the heap (ie: malloc() or equivalent). Hiding the details of the + pool in a separate API also allows for a much simpler bli_mem.c family + of functions. + - Added a new internal header, bli_config_macro_defs.h, which enables + sane defaults for the values previously found in bli_config. Those + values can be overridden by #defining them in bli_config.h the same + way kernel defaults can be overridden in bli_kernel.h. This file most + resembles what was previously a typical configuration's bli_config.h. + - Added a new configuration macro, BLIS_POOL_ADDR_ALIGN_SIZE, which + defaults to BLIS_PAGE_SIZE, to specify the alignment of individual + blocks in the memory pool. Also added a corresponding query routine to + the bli_info API. + - Deprecated (once again) the micro-panel alignment feature. Upon further + reflection, it seems that the goal of more predictable L1 cache + replacement behavior is outweighed by the harm caused by non-contiguous + micro-panels when k % kc != 0. I honestly don't think anyone will even + miss this feature. + - Changed bli_ukr_get_funcs() and bli_ukr_get_ref_funcs() to call + bli_cntl_init() instead of bli_init(). + - Removed query functions from bli_info.c that are no longer applicable + given the dynamic memory allocator. + - Removed unnecessary definitions from configurations' bli_config.h files, + which are now pleasantly sparse. + - Fixed incorrect flop counts in addv, subv, scal2v, scal2m testsuite + modules. Thanks to Devangi Parikh for pointing out these + miscalculations. + - Comment, whitespace changes. + +commit 9848f255a3bab17d1139c391cca13ff3f1ffe6ed +Author: Field G. Van Zee +Date: Thu Jun 11 19:14:22 2015 -0500 + + Added early return to API-level _init() routines. + + Details: + - Added conditional code that returns early from the API-level _init() + routines if the API is already initialized. Actually meant for this to + be included in 5f93cbe8. + +commit 5f93cbe870f3478870e15581e7fd450dad5bba1e +Author: Field G. Van Zee +Date: Thu Jun 11 18:52:12 2015 -0500 + + Introduced API-level initialization. + + Details: + - Added API-level initialization state to _const, _error, _mem, _thread, + _ind, and _cntl APIs. While this functionality will mostly go unused, + adding miniscule overhead at init-time, there will be at least once + instance in the near future where, in order to avoid an infinite loop, + a certain portion of the initialization will call a query function that + itself attempts to call bli_init(). API-level initialization will allow + this later stage to verify that an earlier stage of initialization has + completed, even if the overall call to bli_init() has not yet returned. + - Added _is_initialized() functions for each API, setting the underlying + bool_t during _init() and unsetting it during _finalize(). + - Comment, whitespace changes. + +commit ee129c6b028bc5ac88da7c74fde72c49803742ff +Author: Field G. Van Zee +Date: Wed Jun 10 12:53:28 2015 -0500 + + Fixed bugs in _get_range(), _get_range_weighted(). + + Details: + - Fixed some bugs that only manifested in multithreaded instances of + some (non-gemm) level-3 operations. The bugs were related to invalid + allocation of "edge" cases to thread subpartitions. (Here, we define + an "edge" case to be one where the dimension being partitioned for + parallelism is not a whole multiple of whatever register blocksize + is needed in that dimension.) In BLIS, we always require edge cases + to be part of the bottom, right, or bottom-right subpartitions. + (This is so that zero-padding only has to happen at the bottom, right, + or bottom-right edges of micro-panels.) The previous implementations + of bli_get_range() and _get_range_weighted() did not adhere to this + implicit policy and thus produced bad ranges for some combinations of + operation, parameter cases, problem sizes, and n-way parallelism. + - As part of the above fix, the functions bli_get_range() and + _get_range_weighted() have been renamed to use _l2r, _r2l, _t2b, + and _b2t suffixes, similar to the partitioning functions. This is + an easy way to make sure that the variants are calling the right + version of each function. The function signatures have also been + changed slightly. + - Comment/whitespace updates. + - Removed unnecessary '/' from macros in bli_obj_macro_defs.h. + +commit 9135dfd69d39f3bbd75034f479f27a78dbfebcce +Author: Field G. Van Zee +Date: Fri Jun 5 13:37:44 2015 -0500 + + Minor updates to test/3m4m files. + +commit d62ceece943b20537ec4dd99f25136b9ba2ae340 +Author: Field G. Van Zee +Date: Wed Jun 3 12:56:45 2015 -0500 + + Minor update to test/3m4m/runme.sh. + + Details: + - Removed some stale script code that should have been removed + during 590bb3b8c. + +commit b6ee82a3d421c9c4f1eb6848c7c6e37aa46de799 +Author: Field G. Van Zee +Date: Wed Jun 3 12:14:23 2015 -0500 + + Minor cleanup to bli_init() and friends. + + Details: + - Spun-off initialization of global scalar constants to bli_const_init() + and of threading stuff to bli_thread_init(). + - Added some missing _finalize() functions, even when there is nothing + to do. + +commit 1213f5cebabc1637ce9dd45c4bfa87bb93677c29 +Author: Field G. Van Zee +Date: Tue Jun 2 13:27:47 2015 -0500 + + POSIX thread bugfixes/edits to bli_init.c, _mem.c. + + Details: + - Fixed a sort-of bug in bli_init.c whereby the wrong pthread mutex + was used to lock access to initialization/finalization actions. + But everything worked out okay as long as bli_init() was called by + single-threaded code. + - Changed to static initialization for memory allocator mutex in + bli_mem.c, and moved mutex to that file (from bli_init.c). + - Fixed some type mismatches in bli_threading_pthreads.c that resulted + in compiler warnings. + - Fixed a small memory leak with allocated-but-never-freed (and unused) + pthread_attr_t objects. + - Whitespace changes to bli_init.c and bli_mem.c. + +commit 590bb3b8c5c0389159c5a9451b6c156c5f237e8a +Author: Field G. Van Zee +Date: Sun May 24 16:02:53 2015 -0500 + + Backed-out adjusted dim changes to test/3m4m. + + Details: + - Reverted most changes applied during commit ec25807b. + +commit ec25807b26da943868f0d0517c3720e50181b8f9 +Author: Field G. Van Zee +Date: Fri Apr 10 13:23:50 2015 -0500 + + Tweaks to test/3m4m to test with adjusted dims. + + Details: + - Updated test/3m4m driver files to build test drivers that allow + comparision of real "asm_blis" results to complex "asm_blis" results, + except with the latter's problem sizes adjusted so that problems are + generated with equal flop counts. + +commit 426b6488580a92bf071a62dc319a9c837ce39821 +Author: Field G. Van Zee +Date: Wed Apr 8 15:12:21 2015 -0500 + + Fixed a packing bug that manifested in trsm_r. + + Details: + - Fixed a bug that caused a memory leak in the contiguous memory + allocator. Because packm_init() was using simple aliasing when + a subpartition object was marked as zeros by bli_acquire_mpart_*(), + the "destination" pack object's mem_t entry was being overwritten + by the corresponding field of the "source" object (which was likely + NULL). This prevented the block from being released back to the + memory allocator. But this bug only manifested when changing the + location of packing B from outside the var1 loop to inside the + var3 loop, and only for trsm with triangular B (side = right). The + bug was fixed by changing the type of alias used in packm_init() + when handling zero partition cases. Specifically, we now use + bli_obj_alias_for_packing(), which does not clobber the destination + (pack) object's mem_t field. Thanks to Devangi Parikh for this bug + report. + +commit c84286d5cef48f16d83831baac1f46b9856b9a36 +Author: Field G. Van Zee +Date: Sat Apr 4 15:39:14 2015 -0500 + + More minor tweaks to test/3m4m. + + Details: + - Added a line of output that forces matlab to allocate the entire array + up-front. + - Re-enabled real domain benchmarks in runme.sh, which were temporarily + disabled. + +commit 309717c8ebf4ef1369f15cf41340e13c25b41573 +Author: Field G. Van Zee +Date: Fri Apr 3 19:28:49 2015 -0500 + + More tweaks to test/3m4m, configurations. + + Details: + - Fixed incorrect number of mc_x_kc memory blocks in + sandybridge/bli_config.h. + - Enabled OpenMP multithreding in piledriver/bli_config.h. + - More updates to test/3m4m driver files. + +commit 4baf3b9c69b2f648be9e46e07ccc9859dd675828 +Author: Field G. Van Zee +Date: Fri Apr 3 16:44:32 2015 -0500 + + Tweaked test/3m4m driver, including acml support. + + Details: + - Added ACML support to test/3m4m driver Makefile and runme.sh script. + +commit a32f7c49ca4ea869d2a6c66818780f4321743d67 +Merge: 349e075 4bfd1ce +Author: Field G. Van Zee +Date: Fri Apr 3 08:28:11 2015 -0500 + + Merge pull request #23 from xianyi/master + + Add auto-detecting CPU on configure stage. + +commit 349e075ad6a8e2a1211d94f36d24828c9d44b052 +Author: Field G. Van Zee +Date: Thu Apr 2 18:12:28 2015 -0500 + + Tweaks to sandybridge config, test/3m4m driver. + + Details: + - Enable OpenMP support by default in sandybridge's bli_config.h. + - Reorganized sandybridge's bli_kernel.h. + - Updated 3m4m Makefile, runme.sh to also test MKL implementation. + +commit 4bfd1ce8ca93f93d170dd2715f0a32027b417b46 +Author: Zhang Xianyi +Date: Thu Apr 2 16:40:21 2015 -0500 + + Detect NEON for cortex-a9 and cortex-a15. + +commit aa6eec4f43137057276fe6119bdbfb5c52682527 +Author: Zhang Xianyi +Date: Thu Apr 2 16:03:44 2015 -0500 + + Detect the CPU architecture. Support ARM cores. + + Detect the CPU architecture by compiler's predefined macros. + Then, detect the CPU cores. + + Support detecting x86 and ARM architectures. + +commit 2947cfb749c937b0f62fac36cc92f123bd45b53c +Author: Zhang Xianyi +Date: Wed Apr 1 12:24:00 2015 -0500 + + Add auto-detecting CPU on configure stage. + e.g. /Path_to_BLIS/configure auto + + Now, it only support detecting x86 CPUs. + +commit 26a4b8f6f985597f80e0174990bf541f1d9bafac +Author: Field G. Van Zee +Date: Wed Apr 1 10:44:54 2015 -0500 + + Implemented 3m2, 3m3 induced algorithms (gemm only). + + Details: + - Defined a new "3ms" (separated 3m) pack schema and added appropriate + support in packm_init(), packm_blk_var2(). + - Generalized packm_struc_cxk_3mi to take the imaginary stride (is_p) + as an argument instead of computing it locally. Exception: for trmm, + is_p must be computed locally, since it changes for triangular + packed matrices. Also exposed is_p in interface to dt-specific + packm_blk_var2 (and _var1, even though it does not use imaginary + stride). + - Renamed many functions/variables from _3mi to _3mis to indicate that + they work for either interleaved or separated 3m pack schemas. + - Generalized gemm and herk macro-kernels to pass in imaginary stride + rather than compute them locally. + - Added support for 3m2 and 3m3 algorithms to frame/ind, including 3m2- + and 3m3-specific virtual micro-kernels. + - Added special gemm macro-kernels to support 3m2 and 3m3. + - Added support for 3m2 and 3m3 to testsuite. + - Corrected the type of the panel dimension (pd_) in various macro- + kernels from inc_t to dim_t. + - Renamed many functions defined in bli_blocksize.c. + - Moved most induced-related macro defs from frame/include to + frame/ind/include. + - Updated the _ukernel.c files so that the micro-kernel function pointers + are obtained from the func_t objects rather than the cpp macros that + define the function names. + - Updated test/3m4m driver, Makefile, and run script. + +commit ddf62ba7d2da08225b201585b85e06c967767dea +Author: Tyler Smith +Date: Fri Mar 27 14:27:51 2015 -0500 + + Refuse to free the packm thread info if it uses the single threaded version + +commit 016fc587584d958a0e430a56a5e2c05022ac2f17 +Author: Tyler Smith +Date: Fri Mar 27 14:23:02 2015 -0500 + + Don't free packm thread info if it is null + +commit 00a443c529a60862a57b93e303a0b3212c9b1df4 +Author: Tyler Smith +Date: Fri Mar 27 14:11:07 2015 -0500 + + Use bli_malloc instead of malloc for the thread info paths + +commit f1a6b7d02861ccebdc500ea98778cc0f6cddad17 +Author: Field G. Van Zee +Date: Wed Mar 18 15:37:10 2015 -0500 + + Reorganized code for induced complex methods. + + Details: + - Consolidated most of the code relating to induced complex methods + (e.g. 4mh, 4m1, 3mh, 3m1, etc.) into frame/ind. Induced methods + are now enabled on a per-operation basis. The current "available" + (enabled and implemented) implementation can then be queried on + an operation basis. Micro-kernel func_t objects as well as blksz_t + objects can also be queried in a similar maner. + - Redefined several micro-kernel and operation-related functions in + bli_info_*() API, in accordance with above changes. + - Added mr and nr fields to blksz_t object, which point to the mr + and nr blksz_t objects for each cache blocksize (and are NULL for + register blocksizes). Renamed the sub-blocksize field "sub" to + "mult" since it is really expressing a blocksize multiple. + - Updated bli_*_determine_kc_[fb]() for gemm/hemm/symm, trmm, and + trsm to correctly query mr and nr (for purposes of nudging kc). + - Introduced an enumerated opid_t in bli_type_defs.h that uniquely + identifies an operation. For now, only level-3 id values are defined, + along with a generic, catch-all BLIS_NOID value. + - Reworked testsuite so that all induced methods that are enabled + are tested (one at a time) rather than only testing the first + available method. + - Reformated summary at the beginning of testsuite output so that + blocksize and micro-kernel info is shown for each induced method + that was requested (as well as native execution). + - Reduced the number of columns needed to display non-matlab + testsuite output (from approx. 90 to 80). + +commit 8d5169ccda954e5f72944308a036dcb7ebfc9097 +Author: Field G. Van Zee +Date: Wed Mar 18 11:38:08 2015 -0500 + + Fixed bug in release of mem_t buffer. + + Details: + - Fixed a bug that affects all level-2 and level-3 blocked variants. The + bug only manifested, however, if the packing of operands (A and B in + gemm, for example) spanned multiple nodes in the control tree. Until + recently, the main consumers of packm were level-3 operations, all of + which packed both input operands from blocked variant 1 (B outside of + the loop, and A within the loop). This particular usage masked a flaw + in the code whereby bli_obj_release_pack() would always release the + underlying mem_t buffer (provided it was allocated), even if the buffer + was not allocated in the current variant. This has been fixed by + replacing all calls to bli_obj_release_pack() with calls to a new + function, bli_packm_release(), which takes the same control tree node + argument passed into the object's corresponding call to packm_init() + or packv_init(). bli_packm_release() then proceeds to invoke + bli_obj_release_pack() only if the control tree node indicates that + packing was requested. Thanks to Devangi Parikh for identifying this + bug. + +commit c0acca0f5182ba96fd39c9d10b34a896a6e74206 +Author: Field G. Van Zee +Date: Tue Mar 3 10:56:22 2015 -0600 + + Clarified comments in testsuite input.operations. + +commit 03ba9a6b17861d9e1adc0cf924439c4d7e860d19 +Author: Field G. Van Zee +Date: Tue Feb 24 10:33:28 2015 -0600 + + Removed some 'old' directories. + +commit a86db60ee270cdeb745ae7cf68f9e0becc9f522d +Author: Field G. Van Zee +Date: Mon Feb 23 18:42:39 2015 -0600 + + Extensive renaming of 3m/4m-related files, symbols. + + Details: + - Renamed all remaining 3m/4m packing files and symbols to 3mi/4mi + ('i' for "interleaved"). Similar changes to 3M/4M macros. + - Renamed all 3m/4m files and functions to 3m1/4m1. + - Whitespace changes. + +commit 8cf8da291a0fb2f491f410969a76ec0fbda47faf +Author: Field G. Van Zee +Date: Fri Feb 20 15:24:27 2015 -0600 + + Minor updates to induced complex mode management. + + Details: + - Relocated bli_4mh.c, bli_4mb.c, bli_4m.c, bli_3mh.c, bli_3m.c (and + associated headers) from frame/base to frame/base/induced. + - Added bli_xm.? to frame/base/induced, which implements + bli_xm_is_enabled(), which detects whether ANY induced complex method + is currently enabled. + - The new function bli_xm_is_enabled() is now used in bli_info.c to + detect when an induced complex method is used, so we know when to + return blocksizes from one of the induced methods' blocksize objects. + +commit 411e637ee7d1083a84f58f08938d51e63d7c3c9a +Merge: c2569b8 fc0b771 +Author: Tyler Michael Smith +Date: Fri Feb 20 20:39:25 2015 -0600 + + Merge branch 'master' of http://github.com/flame/blis + +commit c2569b8803d4ccc1d7b6f391713461b51443601d +Author: Tyler Michael Smith +Date: Fri Feb 20 20:38:19 2015 -0600 + + Fixed a memory leak in freeing the thread infos + +commit fc0b771227abf86d81f505b324f69f6e83db1d8f +Author: Field G. Van Zee +Date: Fri Feb 20 11:47:44 2015 -0600 + + Added max(mr,nr) to kc in static mem pools. + + Details: + - Changed the static memory definitions to compute the maximum register + blocksize for each datatype and add it to kc when computing the size + of blocks of A and B. This formally accounts for the nudging of kc + up to a multiple of mr or nr at runtime for triangular operations + (e.g. trmm). + +commit af32e3a608631953ef770341df10a14a991bf290 +Author: Tyler Michael Smith +Date: Thu Feb 19 22:51:11 2015 -0600 + + Fixed a bug with get_range_weighted would return end = 0 for small problem sizes + +commit 441d47542a64e131578d00da7404c1ed387a721c +Author: Field G. Van Zee +Date: Thu Feb 19 17:06:10 2015 -0600 + + Renamed 3m and 4m symbols/macros to 3mi and 4mi. + + Details: + - Renamed several variables and macros from 3m/4m to 3mi/4mi. This is + because those packing schemas were always implicitly "interleaved". + This new naming scheme will make way for new schemas that separate + instead of interleve the real and imaginary (and summed) parts. + - Expanded the pack format sub-field of the pack schema field of the + info_t to 4 bits (from 3). This will allow for more schema types + going forward. + - Removed old _cntl.c files for herk3m, herk4m, trmm3m, trmm4m. + +commit 518a1756ccf02122b96fc437b538604a597df42a +Author: Field G. Van Zee +Date: Thu Feb 19 14:27:09 2015 -0600 + + Fixed indexing bug for trmm3 via 3mh, 4mh. + + Details: + - Fixed a bug that only affected trmm3 when performed via 3mh or 4mh, + whereby micro-panels of the triangular matrix were packed with "dead + space" between them due to failing to adjust for the fact that pointer + arithmetic was occurring in units of complex elements while the data + being packed consisted of real elements. It turns out that the macro- + kernel suffered from the same bug, meaning the panels were actually + being packed and read consistently. The only way I was able to + discover the bug in the first place was because the packed block of A + was overflowing into the beginning of the packed row panel of B using + the sandybridge configuration. + +commit 493087d730f01d5169434f461644e5633f48a42f +Merge: 650d2a6 2502129 +Author: Field G. Van Zee +Date: Wed Feb 18 09:45:51 2015 -0600 + + Merge branch 'master' of github.com:flame/blis + +commit 25021299b670775df8ca9c87910c63d7e74ed946 +Merge: fe2b8d3 f05a576 +Author: Field G. Van Zee +Date: Wed Feb 11 20:03:21 2015 -0600 + + Merge branch 'master' of github.com:flame/blis + +commit fe2b8d39a445ac848686e78c7540fd046cb95492 +Author: Field G. Van Zee +Date: Wed Feb 11 19:33:10 2015 -0600 + + Fixed an obscure bug in 3mh/3m/4mh/4m packing. + + Details: + - Modified bli_packm_blk_var1.c and _var2.c to increase the triangular + case's panel increment by 1 if it would otherwise be odd. This is + particularly necessary in _var2.c when handling the interleaved 3m + or ro/io/rpi pack schemas, since division of an odd number by 2 can + happen if both the panel length and the panel packing dimension + (register packing blocksize) are odd, thus making their product odd. + - Modified bli_packm_init.c so that panel strides are increased by 1 + if they would otherwise be odd, even for non-3m related packing. + - Modified the trmm and trsm macro-kernels so that triangular packed + micro-panels are traversed with this new "increment by 1 if odd" + policy. + - Added sanity checks in trmm and trsm macro-kernels that would result + in an abort() if the conditions that would lead to a "divide odd + integer by 2" scenario ever manifest. + - Defined bli_is_odd(), _is_even() macros in bli_scalar_macro_defs.h. + +commit 650d2a6ff2e593151a296ca86b5214afcc747afc +Author: Field G. Van Zee +Date: Mon Feb 9 14:59:20 2015 -0600 + + Added initial support for imaginary stride. + + Details: + - Added an imaginary stride field ("is") to obj_t. + - Renamed bli_obj_set_incs() macro to bli_obj_set_strides(). + - Defined bli_obj_imag_stride() and bli_obj_set_imag_stride() and + added invocations in key locations. + - Added some basic error-checking related to imaginary stride. + - For now, imaginary stride will not be exposed into the most-used + BLIS APIs such as bli_obj_create(), and certainly not the + computational APIs such as bli_dgemm(). + +commit f05a57634a7c8e3864b25b3335d1194c1ea1aeb9 +Author: Field G. Van Zee +Date: Sun Feb 8 19:40:34 2015 -0600 + + Defined gemm cntl function to query ukrs func_t. + + Details: + - Added a new function, bli_gemm_cntl_ukrs(), that returns the func_t* + for the gemm micro-kernels from the leaf node of the control tree. + This allows all the func_t* fields from higher-level nodes in the tree + to be NULL, which makes the function that builds the control trees + slightly easier to read. + - Call bli_gemm_cntl_ukrs() instead of the cntl_gemm_ukrs() macro in + all bli_*_front() functions (which is needed to apply the row/column + preference optimization). + - In all level-3 bli_*_cntl_init() functions, changed the _obj_create() + function arguments corresponding to the gemm_ukrs fields in higher- + level cntl tree nodes to NULL. + - Removed some old her2k macro-kernels. + +commit cefd3d5d2001264de17cf63dae541f890cb9daaf +Author: Tyler Smith +Date: Thu Feb 5 11:09:12 2015 -0600 + + A couple of functions were incorrectly ifdeffed away on Xeon Phi. Fixed this + +commit 7574c9947d57a19f613880e3b9f62f8c8f6df4ec +Author: Field G. Van Zee +Date: Wed Feb 4 12:11:55 2015 -0600 + + Added basic flop-counting mechanism (level-3 only). + + Details: + - Added optional flop counting to all level-3 front-ends, which is + enabled via BLIS_ENABLE_FLOP_COUNT. The flop count can be + reset at any time via bli_flop_count_reset() and queried via + bli_flop_count(). Caveats: + - flop counts are approximate for her[2]k, syr[2]k, trmm, and + trsm operations; + - flop counts ignore extra flops due to non-unit alpha; + - flop counts do not account for situations where beta is zero. + +commit ceda4f27d1f1bcf19320e09848e0f2e3b9941e6c +Author: Field G. Van Zee +Date: Thu Jan 29 13:22:54 2015 -0600 + + Implemented bli_obj_imag_equals(). + + Details: + - Implemented a new function, bli_obj_imag_equals(), which compares the + imaginary part of the first argument to the second argument, which may + be a BLIS_CONSTANT or of a regular real datatype. + +commit 81114824a05a9053229efd577a8a94a856deda93 +Author: Field G. Van Zee +Date: Tue Jan 6 12:15:21 2015 -0600 + + Minor 4m/3m consolidation to mem_pool_macro_defs.h. + + Details: + - Merged the 4m and 3m definitions in bli_mem_pool_macro_defs.h to + reduce code and improve readability. + +commit 36a9b7b7436d9423ba4de2a9f85cfcd43577b783 +Author: Tyler Michael Smith +Date: Wed Dec 17 21:53:50 2014 +0000 + + reduced the default number of MC by KC blocks for bgq + +commit c60619c7c3568f044a849abbab60209aa7455423 +Author: Field G. Van Zee +Date: Tue Dec 16 17:08:22 2014 -0600 + + Minor tweaks for 3m4m test drivers. + + Details: + - Changed gemm_kc blocksizes to be reduced by two-thirds instead of + half. + - Changed 3m4m/test_gemm.c driver to divide by 3 instead of 2 when + computing the fixed k dimension. + - Fixed runme.sh so that it would use multiple threads for s/dgemm + cases. + +commit c6929ba6a5e6f633a7295e979a2b8df8c7ecdb1b +Author: Field G. Van Zee +Date: Tue Dec 16 11:27:50 2014 -0600 + + Added 4m_1b to test/3m4m test driver and script. + +commit 785d480805fc0d6f4251b5499933515740b6b2a7 +Merge: 9456f33 4156c08 +Author: Field G. Van Zee +Date: Fri Dec 12 14:34:19 2014 -0600 + + Merge branch 'master' of github.com:flame/blis + +commit 9456f330af4617f9ee32972d51f974aa2d84f97b +Author: Field G. Van Zee +Date: Fri Dec 12 14:31:57 2014 -0600 + + Added 4m_1b implementation for gemm. + + Details: + - Added yet another 4m-based implementation for complex domain level-3 + operations. This method, which the 3m/4m paper identifies as Algorithm + "4m_1b" fissures the first loop around the micro-kernel so that the + real sub-panel of the current micro-panel of B is multiplied against + (both sub-panels of) all micro-panels of A, before doing the same for + the imaginary sub-panel of the micro-panel of B. For now, only gemm is + supported, and 4m_1b (labeled "4mb" within the framework) is not yet + integrated into the test suite. + +commit 4156c0880d9aea4ff04a9c4fa139ba8c437d8bfb +Author: Field G. Van Zee +Date: Tue Dec 9 16:03:14 2014 -0600 + + Fixed obscure level-2 packing / general stride bug. + + Details: + - Fixed a bug in certain structured level-2 operations that manifested + only when the structured matrix was provided to BLIS as matrix stored + with general stride. The bug was introduced in c472993b when the + densify field was removed from the packm control tree node and + associated APIs. Since then, the packed object was unconditionally + marked with an uplo field of BLIS_DENSE. This is fine for level-3 + operations where micro-panels are always densified, but in level-2 + contexts, the underlying unblocked variant (fused or unfused) of + structured operations (e.g. trmv) still needs to know whether to + execute its "lower" or "upper" branches of code. Since this field + was unconditionally being set to BLIS_DENSE, the unblocked variants + were always executed the "else" branch, which happened to be the + "lower" case code. Thus, running an upper case produced the wrong + answer. This most obviously manifested in the form of failures for + trmm, trmm3, and trsm in the test suite. + The bug was fixed by setting the packed object's uplo field to + BLIS_DENSE only if the schema indicated that micro-panels were to be + packed. Otherwise, we can assume we are packing to regular row or + column storage, as is the case with level-2 packing. Thanks to + Francisco Igual for reporting the testsuite failures and ultimately + leading us to this bug. + +commit 689f60a578b461119e9ea90c74f642b9eb79addb +Merge: bef24e6 483e4d6 +Author: Field G. Van Zee +Date: Sun Dec 7 14:03:30 2014 -0600 + + Merge pull request #21 from figual/master + + Adding armv8a configuration and micro-kernels. + +commit 483e4d6a3fdbef9d9ab47fb674c9476c70ca9f0f +Author: Francisco D. Igual +Date: Sun Dec 7 20:27:49 2014 +0100 + + Adding armv8a configuration and micro-kernels. + + Only sgemm micro-kernel is fully functional at this point. + +commit bef24e67e0f93579c2a80315348dc2e227f72a72 +Author: Tyler Smith +Date: Wed Nov 26 18:00:56 2014 -0600 + + Fixed a type of race condition exposed by pthreads implementation. + Lead thread of the inner thread communicator could exit subproblem, move on the next iteration of the loop and modify a1_pack, b1_pack, or c1_pack while other threads were still using those. + + Barriers were inserted to fix this. + +commit 76bde44411f0e34266bab9d666a54ef22be97320 +Merge: e56e614 f3d729e +Author: Field G. Van Zee +Date: Wed Nov 26 17:25:24 2014 -0600 + + Merge branch 'master' of github.com:flame/blis + +commit f3d729e504ec012e7dc7e02b2ecd42e004c6894d +Author: Tyler Michael Smith +Date: Wed Nov 26 22:25:24 2014 -0600 + + Added static mutex to bli_init and bli_finalize + +commit d71cc797866ff502ad1127527016f463267eef80 +Author: Tyler Michael Smith +Date: Wed Nov 26 21:35:39 2014 -0600 + + Refactored bli_threading files and added support for pthreads + +commit e56e61438ff7fcf25a48c0b7603f18df782b50b6 +Author: Field G. Van Zee +Date: Wed Nov 26 17:20:35 2014 -0600 + + Minor cleanups to bli_threading.h and friends. + + Details: + - No longer need to define BLIS_ENABLE_MULTITHREADING manually in + bli_config.h; it now gets defined when BLIS_ENABLE_OPENMP or + BLIS_ENABLE_PTHREADS is defined. + - Added sanity check to prevent both BLIS__ENABLE_OPENMP and + BLIS_ENABLE_PTHREADS from being enabled simultaneously. + - Reorganization of bli_threading*.h header files, which led to + simplification of threading-related part of blis.h. + - added "-fopenmp -lpthread" to LDFLAGS of sandybridge make_defs.mk + file. + +commit 3be2744cbe2c56d38c23fd818aa5c1f10cc7ea51 +Author: Field G. Van Zee +Date: Fri Nov 21 12:28:08 2014 -0600 + + Update to template gemm ukernel comments. + + Details: + - Updated comments on alignment of a1 and b1 to match wiki. + +commit 994429c6881b2ade92d9d7949bcaebfbf2cc65eb +Merge: 58796ab 694029d +Author: Field G. Van Zee +Date: Thu Nov 20 13:55:35 2014 -0600 + + Merge pull request #20 from TimmyLiu/master + + #define PASTEF773 required by cblas compatibility layer + +commit 694029d9d7db857d642ab536955c0621791108c8 +Author: Timmy +Date: Wed Nov 19 15:25:14 2014 -0600 + + #define PASTEF773 required by cblas compatiility layer + +commit 58796abda66b133346f8d523b39178afc336351f +Author: Field G. Van Zee +Date: Thu Nov 6 14:31:52 2014 -0600 + + Removed KC constraint comments from _kernel.h files. + + Details: + - Since 4674ca8c, the constraint that KC be a multiple of both MR and + NR have been relaxed, and thus it was time to remove the comments + from the top of the bli_kernel.h files of all configurations. + +commit 7bbc95a54f706d43c7f7951f0e5995f86130cd52 +Author: Field G. Van Zee +Date: Wed Oct 29 10:52:23 2014 -0500 + + Added new piledriver micro-kernels. + + Details: + - Added new micro-kernels for the AMD piledriver architecture (one + for each datatype). + - Updates and tweaks to piledriver configuration. + - Added 3xk packm micro-kernel support. + - Explicitly unrolled some of the smaller packm micro-kernels. + - Added notes to avx/sandybridge and piledriver micro-kernel files + acknowledging the influence of the corresponding kernel code in + OpenBLAS. + +commit 59613f1d5500f6279963327db2fbc84bc9135183 +Author: Field G. Van Zee +Date: Thu Oct 23 17:21:37 2014 -0500 + + Added separeate micro-panel alignment for A and B. + + Details: + - Changed the recently-added micro-panel alignment macros so that we now + have two sets--one for micro-panels of matrix A and one for micro- + panels of matrix B: BLIS_UPANEL_[AB]_ALIGN_SIZE_?. + - Store each set of alignment values into a separate blksz_t object in + bli_gemm_cntl_init(). + - Adjusted packm_init() to use the separate alignment values. + - Added query routines for the new alignment values to bli_info.c. + - Modified test suite output accordingly. + +commit a8e12884ee1fddd3fd77ca5a68aa0cb857f3af57 +Author: Field G. Van Zee +Date: Thu Oct 23 11:35:48 2014 -0500 + + CHANGELOG update (0.1.6) + +commit 38ea5022e4ed846112198c4e1672fcdaeb90dc71 (tag: 0.1.6) Author: Field G. Van Zee Date: Thu Oct 23 11:35:45 2014 -0500 Version file update (0.1.6) -commit a3e6341bdb0e28411f935d6b4708a6389663e004 (origin/master, origin/HEAD) +commit a3e6341bdb0e28411f935d6b4708a6389663e004 Author: Field G. Van Zee Date: Thu Oct 23 11:13:28 2014 -0500 @@ -948,7 +1763,7 @@ Date: Mon Aug 4 16:01:59 2014 -0500 CHANGELOG update (0.1.5) -commit bde56d0ecfd0ec20330fac290b91a6dca0cf94e9 (0.1.5) +commit bde56d0ecfd0ec20330fac290b91a6dca0cf94e9 (tag: 0.1.5) Author: Field G. Van Zee Date: Mon Aug 4 16:01:58 2014 -0500 @@ -1097,7 +1912,7 @@ Date: Sun Jul 27 18:20:13 2014 -0500 CHANGELOG update (0.1.4) -commit a7537071b152ecff671f8716595d37dc09e4fd51 (0.1.4) +commit a7537071b152ecff671f8716595d37dc09e4fd51 (tag: 0.1.4) Author: Field G. Van Zee Date: Sun Jul 27 18:20:12 2014 -0500 @@ -1496,7 +2311,7 @@ Date: Mon Jun 23 13:48:17 2014 -0500 CHANGELOG update (0.1.3) -commit 036cc634918463b1caa0fd89c9a211f2f5639af7 (0.1.3) +commit 036cc634918463b1caa0fd89c9a211f2f5639af7 (tag: 0.1.3) Author: Field G. Van Zee Date: Mon Jun 23 13:48:17 2014 -0500 @@ -1629,7 +2444,7 @@ Date: Thu Jun 5 10:54:16 2014 -0500 CHANGELOG update (for 0.1.2). -commit 00f232f8ed1f7c41619b12ebf779ebe2c3b2d3cd (0.1.2) +commit 00f232f8ed1f7c41619b12ebf779ebe2c3b2d3cd (tag: 0.1.2) Author: Tyler Smith Date: Mon Jun 2 13:40:57 2014 -0500 @@ -2257,7 +3072,7 @@ Date: Tue Feb 25 17:58:42 2014 -0600 CHANGELOG update (for 0.1.1). -commit fde5f1fdece19881f50b142e8611b772a647e6d2 (0.1.1) +commit fde5f1fdece19881f50b142e8611b772a647e6d2 (tag: 0.1.1) Author: Field G. Van Zee Date: Tue Feb 25 13:34:56 2014 -0600 @@ -3171,7 +3986,7 @@ Date: Mon Nov 11 10:15:40 2013 -0600 CHANGELOG update (for 0.1.0). -commit 089048d5895a30221b6b1976c9be93ad6443420d (0.1.0) +commit 089048d5895a30221b6b1976c9be93ad6443420d (tag: 0.1.0) Author: Field G. Van Zee Date: Sat Nov 9 17:18:00 2013 -0600 @@ -3886,7 +4701,7 @@ Date: Fri Jul 19 17:15:03 2013 -0500 CHANGELOG update (for 0.0.9). -commit 0680916fdd532f7a4716b11a2515243b2c08d00f (0.0.9) +commit 0680916fdd532f7a4716b11a2515243b2c08d00f (tag: 0.0.9) Author: Field G. Van Zee Date: Thu Jul 18 18:04:34 2013 -0500 @@ -4124,7 +4939,7 @@ Date: Wed Jun 12 16:40:04 2013 -0500 CHANGELOG update. -commit 5b641c3bab31eac6a1795b9f6e3f86c59651ca50 (0.0.8) +commit 5b641c3bab31eac6a1795b9f6e3f86c59651ca50 (tag: 0.0.8) Author: Field G. Van Zee Date: Wed Jun 12 16:02:12 2013 -0500 @@ -4311,7 +5126,7 @@ Date: Wed May 1 15:00:30 2013 -0500 CHANGELOG update. -commit 6bfa96f84887dec0b4cf8be5d38dd634c2f8951d (0.0.7) +commit 6bfa96f84887dec0b4cf8be5d38dd634c2f8951d (tag: 0.0.7) Author: Field G. Van Zee Date: Tue Apr 30 19:35:54 2013 -0500 @@ -4693,7 +5508,7 @@ Date: Sat Apr 13 16:53:16 2013 -0500 CHANGELOG update. -commit ec16c52f2ecf419c749175ce0a297441c10f1c68 (0.0.6) +commit ec16c52f2ecf419c749175ce0a297441c10f1c68 (tag: 0.0.6) Author: Field G. Van Zee Date: Sat Apr 13 16:41:16 2013 -0500 @@ -5003,7 +5818,7 @@ Date: Sun Mar 24 20:18:12 2013 -0500 CHANGELOG update. -commit b65cdc57d9e51fa00e3c03539cfb7e045707d0f4 (0.0.5) +commit b65cdc57d9e51fa00e3c03539cfb7e045707d0f4 (tag: 0.0.5) Author: Field G. Van Zee Date: Sun Mar 24 20:01:49 2013 -0500 @@ -5107,7 +5922,7 @@ Date: Mon Mar 18 10:37:03 2013 -0500 CHANGELOG update. -commit e7d41229d3b1674e74f47d7f29fae004a745201a (0.0.4) +commit e7d41229d3b1674e74f47d7f29fae004a745201a (tag: 0.0.4) Author: Field G. Van Zee Date: Fri Mar 15 17:12:36 2013 -0500 @@ -5235,7 +6050,7 @@ Date: Fri Feb 22 12:38:45 2013 -0600 configuration directory (bl2_config.h, specifically) given that it can be expected to be tweaked by some developers. -commit ede75693e5a36c6006087c4a7df834175b604504 (0.0.3) +commit ede75693e5a36c6006087c4a7df834175b604504 (tag: 0.0.3) Author: Field G. Van Zee Date: Fri Feb 22 12:11:24 2013 -0600 @@ -5445,7 +6260,7 @@ Date: Mon Feb 11 13:38:07 2013 -0600 CHANGELOG update. -commit 768fcebaa8be0eb936a6e7a02cd8a19438c79d99 (0.0.2) +commit 768fcebaa8be0eb936a6e7a02cd8a19438c79d99 (tag: 0.0.2) Author: Field G. Van Zee Date: Mon Feb 11 13:20:44 2013 -0600 @@ -5687,7 +6502,7 @@ Date: Mon Dec 10 17:23:32 2012 -0600 Minor updates towards to 0.0.1. -commit 7ad4ebef38b8e6eea9b6091844ba7294ec870271 (0.0.1) +commit 7ad4ebef38b8e6eea9b6091844ba7294ec870271 (tag: 0.0.1) Author: Field G. Van Zee Date: Mon Dec 10 16:18:40 2012 -0600 @@ -5755,7 +6570,7 @@ Date: Thu Dec 6 14:27:11 2012 -0600 Wrote first draft of INSTALL file. -commit bcbe81235a35ccfdbcc2f2319a0ca6e04f75a785 (0.0.0) +commit bcbe81235a35ccfdbcc2f2319a0ca6e04f75a785 (tag: 0.0.0) Author: Field G. Van Zee Date: Thu Dec 6 12:42:35 2012 -0600 From d4b891369c1eb0879ade662ff896a5b9a7fca207 Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Tue, 7 Jul 2015 10:06:53 -0500 Subject: [PATCH 12/38] Added 'carrizo' configuration. Details: - Added a new configuration for AMD Excavator-based hardware also known as Carrizo when referring to the entire APU. This configuration uses the same micro-kernels as the piledriver, but with different cache blocksizes. --- config/carrizo/bli_config.h | 48 +++++++++++ config/carrizo/bli_kernel.h | 165 ++++++++++++++++++++++++++++++++++++ config/carrizo/kernels | 1 + config/carrizo/make_defs.mk | 110 ++++++++++++++++++++++++ 4 files changed, 324 insertions(+) create mode 100644 config/carrizo/bli_config.h create mode 100644 config/carrizo/bli_kernel.h create mode 120000 config/carrizo/kernels create mode 100644 config/carrizo/make_defs.mk diff --git a/config/carrizo/bli_config.h b/config/carrizo/bli_config.h new file mode 100644 index 0000000000..b0da5de467 --- /dev/null +++ b/config/carrizo/bli_config.h @@ -0,0 +1,48 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef BLIS_CONFIG_H +#define BLIS_CONFIG_H + + +//#define BLIS_ENABLE_PTHREADS +#define BLIS_ENABLE_OPENMP + +#define BLIS_SIMD_ALIGN_SIZE 16 + + + + + +#endif diff --git a/config/carrizo/bli_kernel.h b/config/carrizo/bli_kernel.h new file mode 100644 index 0000000000..cdd1301b84 --- /dev/null +++ b/config/carrizo/bli_kernel.h @@ -0,0 +1,165 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef BLIS_KERNEL_H +#define BLIS_KERNEL_H + + +// -- LEVEL-3 MICRO-KERNEL CONSTANTS ------------------------------------------- + +// -- Cache blocksizes -- + +// +// Constraints: +// +// (1) MC must be a multiple of: +// (a) MR (for zero-padding purposes) +// (b) NR (for zero-padding purposes when MR and NR are "swapped") +// (2) NC must be a multiple of +// (a) NR (for zero-padding purposes) +// (b) MR (for zero-padding purposes when MR and NR are "swapped") +// + +#define BLIS_SGEMM_UKERNEL bli_sgemm_new_16x3 +#define BLIS_DEFAULT_MC_S 528 +#define BLIS_DEFAULT_KC_S 256 +#define BLIS_DEFAULT_NC_S 8400 +#define BLIS_DEFAULT_MR_S 16 +#define BLIS_DEFAULT_NR_S 3 + +#define BLIS_DGEMM_UKERNEL bli_dgemm_new_8x3 +#define BLIS_DEFAULT_MC_D 264 +#define BLIS_DEFAULT_KC_D 256 +#define BLIS_DEFAULT_NC_D 8400 +#define BLIS_DEFAULT_MR_D 8 +#define BLIS_DEFAULT_NR_D 3 + +#define BLIS_CGEMM_UKERNEL bli_cgemm_new_4x2 +#define BLIS_DEFAULT_MC_C 264 +#define BLIS_DEFAULT_KC_C 256 +#define BLIS_DEFAULT_NC_C 8400 +#define BLIS_DEFAULT_MR_C 4 +#define BLIS_DEFAULT_NR_C 2 + +#define BLIS_ZGEMM_UKERNEL bli_zgemm_new_2x2 +#define BLIS_DEFAULT_MC_Z 100 +#define BLIS_DEFAULT_KC_Z 320 +#define BLIS_DEFAULT_NC_Z 8400 +#define BLIS_DEFAULT_MR_Z 2 +#define BLIS_DEFAULT_NR_Z 2 + + + +// -- Register blocksizes -- + + + +// -- Micro-panel alignment -- + + + + + +// -- LEVEL-2 KERNEL CONSTANTS ------------------------------------------------- + + + + +// -- LEVEL-1F KERNEL CONSTANTS ------------------------------------------------ + + + + +// -- LEVEL-3 KERNEL DEFINITIONS ----------------------------------------------- + +// -- gemm -- + + +// -- trsm-related -- + + + + +// -- LEVEL-1M KERNEL DEFINITIONS ---------------------------------------------- + +// -- packm -- + +// -- unpackm -- + + + + +// -- LEVEL-1F KERNEL DEFINITIONS ---------------------------------------------- + +// -- axpy2v -- + +// -- dotaxpyv -- + +// -- axpyf -- + +// -- dotxf -- + +// -- dotxaxpyf -- + + + + +// -- LEVEL-1V KERNEL DEFINITIONS ---------------------------------------------- + +// -- addv -- + +// -- axpyv -- + +// -- copyv -- + +// -- dotv -- + +// -- dotxv -- + +// -- invertv -- + +// -- scal2v -- + +// -- scalv -- + +// -- setv -- + +// -- subv -- + +// -- swapv -- + + + +#endif + diff --git a/config/carrizo/kernels b/config/carrizo/kernels new file mode 120000 index 0000000000..a3f3e222d4 --- /dev/null +++ b/config/carrizo/kernels @@ -0,0 +1 @@ +../../kernels/x86_64/piledriver \ No newline at end of file diff --git a/config/carrizo/make_defs.mk b/config/carrizo/make_defs.mk new file mode 100644 index 0000000000..5f5303ade3 --- /dev/null +++ b/config/carrizo/make_defs.mk @@ -0,0 +1,110 @@ +#!/bin/bash +# +# BLIS +# An object-based framework for developing high-performance BLAS-like +# libraries. +# +# Copyright (C) 2014, The University of Texas at Austin +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# - Neither the name of The University of Texas at Austin nor the names +# of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# Only include this block of code once. +ifndef MAKE_DEFS_MK_INCLUDED +MAKE_DEFS_MK_INCLUDED := yes + + + +# +# --- Build definitions -------------------------------------------------------- +# + +# Variables corresponding to other configure-time options. +BLIS_ENABLE_VERBOSE_MAKE_OUTPUT := no +BLIS_ENABLE_STATIC_BUILD := yes +BLIS_ENABLE_DYNAMIC_BUILD := no + + + +# +# --- Utility program definitions ---------------------------------------------- +# + +SH := /bin/sh +MV := mv +MKDIR := mkdir -p +RM_F := rm -f +RM_RF := rm -rf +SYMLINK := ln -sf +FIND := find +GREP := grep +XARGS := xargs +RANLIB := ranlib +INSTALL := install -c + +# Used to refresh CHANGELOG. +GIT := git +GIT_LOG := $(GIT) log --decorate + + + +# +# --- Development tools definitions -------------------------------------------- +# + +# --- Determine the C compiler and related flags --- +CC := gcc +# Enable IEEE Standard 1003.1-2004 (POSIX.1d). +# NOTE: This is needed to enable posix_memalign(). +CPPROCFLAGS := -D_POSIX_C_SOURCE=200112L +CMISCFLAGS := -std=c99 -fopenmp +CPICFLAGS := -fPIC +CDBGFLAGS := #-g +CWARNFLAGS := -Wall +COPTFLAGS := -O2 -mfpmath=sse -fomit-frame-pointer +CKOPTFLAGS := $(COPTFLAGS) +CVECFLAGS := -mavx -mfma -march=native + +# Aggregate all of the flags into multiple groups: one for standard +# compilation, and one for each of the supported "special" compilation +# modes. +CFLAGS_NOOPT := $(CDBGFLAGS) $(CWARNFLAGS) $(CPICFLAGS) $(CMISCFLAGS) $(CPPROCFLAGS) +CFLAGS := $(COPTFLAGS) $(CVECFLAGS) $(CFLAGS_NOOPT) +CFLAGS_KERNELS := $(CKOPTFLAGS) $(CVECFLAGS) $(CFLAGS_NOOPT) + +# --- Determine the archiver and related flags --- +AR := ar +ARFLAGS := cru + +# --- Determine the linker and related flags --- +LINKER := $(CC) +SOFLAGS := -shared +LDFLAGS := -lm -fopenmp + + + +# end of ifndef MAKE_DEFS_MK_INCLUDED conditional block +endif From fdfe14f1e17ba5a2f8dfa0bdb799c6b0e730211b Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Thu, 9 Jul 2015 13:52:39 -0500 Subject: [PATCH 13/38] Added support for Intel Haswell/Broadwell. Details: - Added sgemm and dgemm micro-kernels, which employ 256-bit AVX vectors and FMA instructions. (Complex support is currently provided by default induced method, 4m1a.) - Added a 'haswell' configuration, which uses the aforementioned kernels. - Inserted auto-detection support for haswell configuration in build/auto-detect/cpuid_x86.c. - Modified configure script to explicitly echo when automatic or manual configuration is in progress. - Changed beta scalar in test_gemm.c module of test suite to -1.0 to 0.9. --- build/auto-detect/cpuid_x86.c | 149 +-- config/haswell/bli_config.h | 46 + config/haswell/bli_kernel.h | 172 +++ config/haswell/kernels | 1 + config/haswell/make_defs.mk | 110 ++ configure | 18 +- kernels/x86_64/avx2/3/bli_gemm_asm_d12x4.c | 1338 ++++++++++++++++++++ kernels/x86_64/avx2/3/bli_gemm_asm_d8x6.c | 1326 +++++++++++++++++++ testsuite/src/test_gemm.c | 4 +- 9 files changed, 3085 insertions(+), 79 deletions(-) create mode 100644 config/haswell/bli_config.h create mode 100644 config/haswell/bli_kernel.h create mode 120000 config/haswell/kernels create mode 100644 config/haswell/make_defs.mk create mode 100644 kernels/x86_64/avx2/3/bli_gemm_asm_d12x4.c create mode 100644 kernels/x86_64/avx2/3/bli_gemm_asm_d8x6.c diff --git a/build/auto-detect/cpuid_x86.c b/build/auto-detect/cpuid_x86.c index e7757862b9..6bbaf9f4e4 100644 --- a/build/auto-detect/cpuid_x86.c +++ b/build/auto-detect/cpuid_x86.c @@ -42,13 +42,15 @@ #define CPUNAME_REFERENCE 0 #define CPUNAME_DUNNINGTON 1 #define CPUNAME_SANDYBRIDGE 2 -#define CPUNAME_BULLDOZER 3 -#define CPUNAME_PILEDRIVER 4 +#define CPUNAME_HASWELL 3 +#define CPUNAME_BULLDOZER 4 +#define CPUNAME_PILEDRIVER 5 static char *cpuname[] = { "reference", "dunnington", "sandybridge", + "haswell", "bulldozer", "piledriver", }; @@ -122,72 +124,73 @@ int cpu_detect() int eax, ebx, ecx, edx; int vendor, family, extend_family, model, extend_model; - if (!have_cpuid()) return CPUNAME_REFERENCE; + if ( !have_cpuid() ) return CPUNAME_REFERENCE; - vendor=get_vendor(); + vendor = get_vendor(); - cpuid(1, &eax, &ebx, &ecx, &edx); - extend_family = BITMASK(eax, 20, 0xff); - extend_model=BITMASK(eax, 16, 0x0f); - family=BITMASK(eax, 8, 0x0f); - model=BITMASK(eax, 4, 0x0f); + cpuid( 1, &eax, &ebx, &ecx, &edx ); + + extend_family = BITMASK( eax, 20, 0xff ); + extend_model = BITMASK( eax, 16, 0x0f ); + family = BITMASK( eax, 8, 0x0f ); + model = BITMASK( eax, 4, 0x0f ); if (vendor == VENDOR_INTEL){ switch (family) { case 0x6: switch (extend_model) { case 1: - switch (model) { - case 7: - //penryn uses dunnington config. - return CPUNAME_DUNNINGTON; - case 13: - return CPUNAME_DUNNINGTON; - } - break; + switch (model) { + case 7: + //penryn uses dunnington config. + return CPUNAME_DUNNINGTON; + case 13: + return CPUNAME_DUNNINGTON; + } + break; case 2: - switch (model) { - case 10: - case 13: - if(support_avx()) { - return CPUNAME_SANDYBRIDGE; - }else{ - return CPUNAME_REFERENCE; //OS doesn't support AVX - } - } - break; + switch (model) { + case 10: + case 13: + if(support_avx()) { + return CPUNAME_SANDYBRIDGE; + }else{ + return CPUNAME_REFERENCE; //OS doesn't support AVX + } + } + break; case 3: - switch (model) { - case 10: - case 14: - //Ivy Bridge - if(support_avx()) { - return CPUNAME_SANDYBRIDGE; - }else{ - return CPUNAME_REFERENCE; //OS doesn't support AVX - } - case 12: - case 15: - //Haswell. Temp use Sandy Brdige - if(support_avx()) { - return CPUNAME_SANDYBRIDGE; - }else{ - return CPUNAME_REFERENCE; //OS doesn't support AVX - } - - } - break; + switch (model) { + case 10: + case 14: + //Ivy Bridge + if(support_avx()) { + return CPUNAME_SANDYBRIDGE; + }else{ + return CPUNAME_REFERENCE; //OS doesn't support AVX + } + case 12: + case 15: + //Haswell + if(support_avx()) { + return CPUNAME_HASWELL; + }else{ + return CPUNAME_REFERENCE; //OS doesn't support AVX + } + + } + break; case 4: switch (model) { case 5: - case 6: - //Haswell. Temp use Sandy Brdige - if(support_avx()) { - return CPUNAME_SANDYBRIDGE; - }else{ - return CPUNAME_REFERENCE; //OS doesn't support AVX - } - } + case 6: + //Haswell + if(support_avx()) { + return CPUNAME_HASWELL; + }else{ + return CPUNAME_REFERENCE; //OS doesn't support AVX + } + } break; } break; @@ -197,24 +200,24 @@ int cpu_detect() case 0xf: switch (extend_family) { case 6: - switch (model) { - case 1: - if(support_avx()) - return CPUNAME_BULLDOZER; - else - return CPUNAME_REFERENCE; //OS don't support AVX. - case 2: - if(support_avx()) - return CPUNAME_PILEDRIVER; - else - return CPUNAME_REFERENCE; //OS don't support AVX. - case 0: - //Steamroller. Temp use Piledriver. - if(support_avx()) - return CPUNAME_PILEDRIVER; - else - return CPUNAME_REFERENCE; //OS don't support AVX. - } + switch (model) { + case 1: + if(support_avx()) + return CPUNAME_BULLDOZER; + else + return CPUNAME_REFERENCE; //OS don't support AVX. + case 2: + if(support_avx()) + return CPUNAME_PILEDRIVER; + else + return CPUNAME_REFERENCE; //OS don't support AVX. + case 0: + //Steamroller. Temp use Piledriver. + if(support_avx()) + return CPUNAME_PILEDRIVER; + else + return CPUNAME_REFERENCE; //OS don't support AVX. + } } break; } diff --git a/config/haswell/bli_config.h b/config/haswell/bli_config.h new file mode 100644 index 0000000000..5f66f6dae4 --- /dev/null +++ b/config/haswell/bli_config.h @@ -0,0 +1,46 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef BLIS_CONFIG_H +#define BLIS_CONFIG_H + +// Enable multithreading via POSIX threads. +//#define BLIS_ENABLE_PTHREADS + +// Enable multithreading via OpenMP. +#define BLIS_ENABLE_OPENMP + + + +#endif diff --git a/config/haswell/bli_kernel.h b/config/haswell/bli_kernel.h new file mode 100644 index 0000000000..9df503005e --- /dev/null +++ b/config/haswell/bli_kernel.h @@ -0,0 +1,172 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef BLIS_KERNEL_H +#define BLIS_KERNEL_H + + +// -- LEVEL-3 MICRO-KERNEL CONSTANTS AND DEFINITIONS --------------------------- + +// +// Constraints: +// +// (1) MC must be a multiple of: +// (a) MR (for zero-padding purposes) +// (b) NR (for zero-padding purposes when MR and NR are "swapped") +// (2) NC must be a multiple of +// (a) NR (for zero-padding purposes) +// (b) MR (for zero-padding purposes when MR and NR are "swapped") +// + +#if 0 + +#define BLIS_SGEMM_UKERNEL bli_sgemm_asm_24x4 +#define BLIS_DEFAULT_MC_S 264 +#define BLIS_DEFAULT_KC_S 128 +#define BLIS_DEFAULT_NC_S 4080 +#define BLIS_DEFAULT_MR_S 24 +#define BLIS_DEFAULT_NR_S 4 + +#else + +#define BLIS_SGEMM_UKERNEL bli_sgemm_asm_16x6 +#define BLIS_DEFAULT_MC_S 144 +#define BLIS_DEFAULT_KC_S 256 +#define BLIS_DEFAULT_NC_S 4080 +#define BLIS_DEFAULT_MR_S 16 +#define BLIS_DEFAULT_NR_S 6 + +#endif + +#if 0 + +#define BLIS_DGEMM_UKERNEL bli_dgemm_asm_12x4 +#define BLIS_DEFAULT_MC_D 96 +#define BLIS_DEFAULT_KC_D 192 +#define BLIS_DEFAULT_NC_D 4080 +#define BLIS_DEFAULT_MR_D 12 +#define BLIS_DEFAULT_NR_D 4 + +#else + +#define BLIS_DGEMM_UKERNEL bli_dgemm_asm_8x6 +#define BLIS_DEFAULT_MC_D 72 +#define BLIS_DEFAULT_KC_D 256 +#define BLIS_DEFAULT_NC_D 4080 +#define BLIS_DEFAULT_MR_D 8 +#define BLIS_DEFAULT_NR_D 6 + +#endif + +/* +#define BLIS_CGEMM_UKERNEL bli_cgemm_asm_8x4 +#define BLIS_DEFAULT_MC_C 96 +#define BLIS_DEFAULT_KC_C 256 +#define BLIS_DEFAULT_NC_C 4096 +#define BLIS_DEFAULT_MR_C 8 +#define BLIS_DEFAULT_NR_C 4 + +#define BLIS_ZGEMM_UKERNEL bli_zgemm_asm_4x4 +#define BLIS_DEFAULT_MC_Z 64 +#define BLIS_DEFAULT_KC_Z 192 +#define BLIS_DEFAULT_NC_Z 4096 +#define BLIS_DEFAULT_MR_Z 4 +#define BLIS_DEFAULT_NR_Z 4 +*/ + + + + +// -- LEVEL-2 KERNEL CONSTANTS ------------------------------------------------- + + + + +// -- LEVEL-1F KERNEL CONSTANTS ------------------------------------------------ + + + + +// -- LEVEL-1M KERNEL DEFINITIONS ---------------------------------------------- + +// -- packm -- + +// -- unpackm -- + + + + +// -- LEVEL-1F KERNEL DEFINITIONS ---------------------------------------------- + +// -- axpy2v -- + +// -- dotaxpyv -- + +// -- axpyf -- + +// -- dotxf -- + +// -- dotxaxpyf -- + + + + +// -- LEVEL-1V KERNEL DEFINITIONS ---------------------------------------------- + +// -- addv -- + +// -- axpyv -- + +// -- copyv -- + +// -- dotv -- + +// -- dotxv -- + +// -- invertv -- + +// -- scal2v -- + +// -- scalv -- + +// -- setv -- + +// -- subv -- + +// -- swapv -- + + + +#endif + diff --git a/config/haswell/kernels b/config/haswell/kernels new file mode 120000 index 0000000000..58026848ba --- /dev/null +++ b/config/haswell/kernels @@ -0,0 +1 @@ +../../kernels/x86_64/avx2 \ No newline at end of file diff --git a/config/haswell/make_defs.mk b/config/haswell/make_defs.mk new file mode 100644 index 0000000000..a854c47c57 --- /dev/null +++ b/config/haswell/make_defs.mk @@ -0,0 +1,110 @@ +#!/bin/bash +# +# BLIS +# An object-based framework for developing high-performance BLAS-like +# libraries. +# +# Copyright (C) 2014, The University of Texas at Austin +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# - Neither the name of The University of Texas at Austin nor the names +# of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# Only include this block of code once. +ifndef MAKE_DEFS_MK_INCLUDED +MAKE_DEFS_MK_INCLUDED := yes + + + +# +# --- Build definitions -------------------------------------------------------- +# + +# Variables corresponding to other configure-time options. +BLIS_ENABLE_VERBOSE_MAKE_OUTPUT := no +BLIS_ENABLE_STATIC_BUILD := yes +BLIS_ENABLE_DYNAMIC_BUILD := no + + + +# +# --- Utility program definitions ---------------------------------------------- +# + +SH := /bin/sh +MV := mv +MKDIR := mkdir -p +RM_F := rm -f +RM_RF := rm -rf +SYMLINK := ln -sf +FIND := find +GREP := grep +XARGS := xargs +RANLIB := ranlib +INSTALL := install -c + +# Used to refresh CHANGELOG. +GIT := git +GIT_LOG := $(GIT) log --decorate + + + +# +# --- Development tools definitions -------------------------------------------- +# + +# --- Determine the C compiler and related flags --- +CC := gcc +# Enable IEEE Standard 1003.1-2004 (POSIX.1d). +# NOTE: This is needed to enable posix_memalign(). +CPPROCFLAGS := -D_POSIX_C_SOURCE=200112L +CMISCFLAGS := -std=c99 -m64 -fopenmp # -fopenmp -pg +CPICFLAGS := -fPIC +CDBGFLAGS := #-g +CWARNFLAGS := -Wall +COPTFLAGS := -O3 -march=native +CKOPTFLAGS := $(COPTFLAGS) +CVECFLAGS := -mavx2 -mfma -mfpmath=sse #-msse3 -march=native # -mfpmath=sse + +# Aggregate all of the flags into multiple groups: one for standard +# compilation, and one for each of the supported "special" compilation +# modes. +CFLAGS_NOOPT := $(CDBGFLAGS) $(CWARNFLAGS) $(CPICFLAGS) $(CMISCFLAGS) $(CPPROCFLAGS) +CFLAGS := $(COPTFLAGS) $(CVECFLAGS) $(CFLAGS_NOOPT) +CFLAGS_KERNELS := $(CKOPTFLAGS) $(CVECFLAGS) $(CFLAGS_NOOPT) + +# --- Determine the archiver and related flags --- +AR := ar +ARFLAGS := cru + +# --- Determine the linker and related flags --- +LINKER := $(CC) +SOFLAGS := -shared +LDFLAGS := -lm -fopenmp -lpthread + + + +# end of ifndef MAKE_DEFS_MK_INCLUDED conditional block +endif diff --git a/configure b/configure index 3c8dbbd278..aa36fa7868 100755 --- a/configure +++ b/configure @@ -145,6 +145,9 @@ main() # Option variables. quiet_flag='' + # The path to the auto-detection script. + auto_detect_sh="${build_dirpath}/auto-detect/auto-detect.sh" + # The name of the reference configuration sub-directory. config_name_ref='reference' @@ -216,11 +219,18 @@ main() elif [ $# = "1" ]; then - if [ $1 = "auto" ]; then - config_name=`${build_dirpath}/auto-detect/auto-detect.sh` - else - config_name=$1 + if [ $1 = "auto" ]; then + + echo "${script_name}: automatic configuration requested." + config_name=`${build_dirpath}/auto-detect/auto-detect.sh` + #config_name=$(${auto_detect_sh}) + + else + + echo "${script_name}: manual configuration requested." + config_name=$1 fi + echo "${script_name}: configuring with '${config_name}' configuration sub-directory." else diff --git a/kernels/x86_64/avx2/3/bli_gemm_asm_d12x4.c b/kernels/x86_64/avx2/3/bli_gemm_asm_d12x4.c new file mode 100644 index 0000000000..09fab6b473 --- /dev/null +++ b/kernels/x86_64/avx2/3/bli_gemm_asm_d12x4.c @@ -0,0 +1,1338 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + + +#define SGEMM_INPUT_GS_BETA_NZ \ + "vmovlps (%%rcx ), %%xmm0, %%xmm0 \n\t" \ + "vmovhps (%%rcx,%%rsi,1), %%xmm0, %%xmm0 \n\t" \ + "vmovlps (%%rcx,%%rsi,2), %%xmm1, %%xmm1 \n\t" \ + "vmovhps (%%rcx,%%r13 ), %%xmm1, %%xmm1 \n\t" \ + "vshufps $0x88, %%xmm1, %%xmm0, %%xmm0 \n\t" \ + "vmovlps (%%rcx,%%rsi,4), %%xmm2, %%xmm2 \n\t" \ + "vmovhps (%%rcx,%%r15 ), %%xmm2, %%xmm2 \n\t" \ + "vmovlps (%%rcx,%%r13,2), %%xmm1, %%xmm1 \n\t" \ + "vmovhps (%%rcx,%%r10 ), %%xmm1, %%xmm1 \n\t" \ + "vshufps $0x88, %%xmm1, %%xmm2, %%xmm2 \n\t" \ + "vperm2f128 $0x20, %%ymm2, %%ymm0, %%ymm0 \n\t" + +#define SGEMM_OUTPUT_GS_BETA_NZ \ + "vextractf128 $1, %%ymm0, %%xmm2 \n\t" \ + "vmovss %%xmm0, (%%rcx ) \n\t" \ + "vpermilps $0x39, %%xmm0, %%xmm1 \n\t" \ + "vmovss %%xmm1, (%%rcx,%%rsi,1) \n\t" \ + "vpermilps $0x39, %%xmm1, %%xmm0 \n\t" \ + "vmovss %%xmm0, (%%rcx,%%rsi,2) \n\t" \ + "vpermilps $0x39, %%xmm0, %%xmm1 \n\t" \ + "vmovss %%xmm1, (%%rcx,%%r13 ) \n\t" \ + "vmovss %%xmm2, (%%rcx,%%rsi,4) \n\t" \ + "vpermilps $0x39, %%xmm2, %%xmm1 \n\t" \ + "vmovss %%xmm1, (%%rcx,%%r15 ) \n\t" \ + "vpermilps $0x39, %%xmm1, %%xmm2 \n\t" \ + "vmovss %%xmm2, (%%rcx,%%r13,2) \n\t" \ + "vpermilps $0x39, %%xmm2, %%xmm1 \n\t" \ + "vmovss %%xmm1, (%%rcx,%%r10 ) \n\t" + +void bli_sgemm_asm_24x4( + dim_t k, + float* restrict alpha, + float* restrict a, + float* restrict b, + float* restrict beta, + float* restrict c, inc_t rs_c, inc_t cs_c, + auxinfo_t* data + ) +{ + //void* a_next = bli_auxinfo_next_a( data ); + //void* b_next = bli_auxinfo_next_b( data ); + + dim_t k_iter = k / 4; + dim_t k_left = k % 4; + + __asm__ volatile + ( + " \n\t" + " \n\t" + "movq %2, %%rax \n\t" // load address of a. + "movq %3, %%rbx \n\t" // load address of b. + //"movq %9, %%r15 \n\t" // load address of b_next. + " \n\t" + " \n\t" // initialize loop by pre-loading + "vmovaps 0 * 32(%%rax), %%ymm0 \n\t" + "vmovaps 1 * 32(%%rax), %%ymm1 \n\t" + "vmovaps 2 * 32(%%rax), %%ymm2 \n\t" + " \n\t" + "movq %6, %%rcx \n\t" // load address of c + "movq %8, %%rdi \n\t" // load cs_c + "leaq (,%%rdi,4), %%rdi \n\t" // cs_c *= sizeof(float) + " \n\t" + "leaq (%%rdi,%%rdi,2), %%r13 \n\t" // r13 = 3*cs_c; + "prefetcht0 7 * 4(%%rcx) \n\t" // prefetch c + 0*cs_c + "prefetcht0 7 * 4(%%rcx,%%rdi) \n\t" // prefetch c + 1*cs_c + "prefetcht0 7 * 4(%%rcx,%%rdi,2) \n\t" // prefetch c + 2*cs_c + "prefetcht0 7 * 4(%%rcx,%%r13) \n\t" // prefetch c + 3*cs_c + " \n\t" + "vxorps %%ymm4, %%ymm4, %%ymm4 \n\t" + "vxorps %%ymm5, %%ymm5, %%ymm5 \n\t" + "vxorps %%ymm6, %%ymm6, %%ymm6 \n\t" + "vxorps %%ymm7, %%ymm7, %%ymm7 \n\t" + "vxorps %%ymm8, %%ymm8, %%ymm8 \n\t" + "vxorps %%ymm9, %%ymm9, %%ymm9 \n\t" + "vxorps %%ymm10, %%ymm10, %%ymm10 \n\t" + "vxorps %%ymm11, %%ymm11, %%ymm11 \n\t" + "vxorps %%ymm12, %%ymm12, %%ymm12 \n\t" + "vxorps %%ymm13, %%ymm13, %%ymm13 \n\t" + "vxorps %%ymm14, %%ymm14, %%ymm14 \n\t" + "vxorps %%ymm15, %%ymm15, %%ymm15 \n\t" + " \n\t" + " \n\t" + " \n\t" + "movq %0, %%rsi \n\t" // i = k_iter; + "testq %%rsi, %%rsi \n\t" // check i via logical AND. + "je .SCONSIDKLEFT \n\t" // if i == 0, jump to code that + " \n\t" // contains the k_left loop. + " \n\t" + " \n\t" + ".SLOOPKITER: \n\t" // MAIN LOOP + " \n\t" + " \n\t" + " \n\t" // iteration 0 + "prefetcht0 16 * 32(%%rax) \n\t" + " \n\t" + "vbroadcastss 0 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm4 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm5 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm6 \n\t" + " \n\t" + "vbroadcastss 1 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm7 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm8 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm9 \n\t" + " \n\t" + "vbroadcastss 2 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm11 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm12 \n\t" + " \n\t" + "vbroadcastss 3 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm13 \n\t" + "vmovaps 3 * 32(%%rax), %%ymm0 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm14 \n\t" + "vmovaps 4 * 32(%%rax), %%ymm1 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm15 \n\t" + "vmovaps 5 * 32(%%rax), %%ymm2 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" // iteration 1 + "vbroadcastss 4 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm4 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm5 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm6 \n\t" + " \n\t" + "vbroadcastss 5 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm7 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm8 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm9 \n\t" + " \n\t" + "vbroadcastss 6 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm11 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm12 \n\t" + " \n\t" + "vbroadcastss 7 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm13 \n\t" + "vmovaps 6 * 32(%%rax), %%ymm0 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm14 \n\t" + "vmovaps 7 * 32(%%rax), %%ymm1 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm15 \n\t" + "vmovaps 8 * 32(%%rax), %%ymm2 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" // iteration 2 + "prefetcht0 22 * 32(%%rax) \n\t" + " \n\t" + "vbroadcastss 8 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm4 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm5 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm6 \n\t" + " \n\t" + "vbroadcastss 9 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm7 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm8 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm9 \n\t" + " \n\t" + "vbroadcastss 10 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm11 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm12 \n\t" + " \n\t" + "vbroadcastss 11 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm13 \n\t" + "vmovaps 9 * 32(%%rax), %%ymm0 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm14 \n\t" + "vmovaps 10 * 32(%%rax), %%ymm1 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm15 \n\t" + "vmovaps 11 * 32(%%rax), %%ymm2 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" // iteration 3 + "vbroadcastss 12 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm4 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm5 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm6 \n\t" + " \n\t" + "vbroadcastss 13 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm7 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm8 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm9 \n\t" + " \n\t" + "vbroadcastss 14 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm11 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm12 \n\t" + " \n\t" + "vbroadcastss 15 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm13 \n\t" + "vmovaps 12 * 32(%%rax), %%ymm0 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm14 \n\t" + "vmovaps 13 * 32(%%rax), %%ymm1 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm15 \n\t" + "vmovaps 14 * 32(%%rax), %%ymm2 \n\t" + " \n\t" + " \n\t" + " \n\t" + "addq $4 * 24 * 4, %%rax \n\t" // a += 4*24 (unroll x mr) + "addq $4 * 4 * 4, %%rbx \n\t" // b += 4*4 (unroll x nr) + " \n\t" + " \n\t" + "decq %%rsi \n\t" // i -= 1; + "jne .SLOOPKITER \n\t" // iterate again if i != 0. + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + ".SCONSIDKLEFT: \n\t" + " \n\t" + "movq %1, %%rsi \n\t" // i = k_left; + "testq %%rsi, %%rsi \n\t" // check i via logical AND. + "je .SPOSTACCUM \n\t" // if i == 0, we're done; jump to end. + " \n\t" // else, we prepare to enter k_left loop. + " \n\t" + " \n\t" + ".SLOOPKLEFT: \n\t" // EDGE LOOP + " \n\t" + "prefetcht0 16 * 32(%%rax) \n\t" + " \n\t" + "vbroadcastss 0 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm4 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm5 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm6 \n\t" + " \n\t" + "vbroadcastss 1 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm7 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm8 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm9 \n\t" + " \n\t" + "vbroadcastss 2 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm11 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm12 \n\t" + " \n\t" + "vbroadcastss 3 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm13 \n\t" + "vmovaps 3 * 32(%%rax), %%ymm0 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm14 \n\t" + "vmovaps 4 * 32(%%rax), %%ymm1 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm15 \n\t" + "vmovaps 5 * 32(%%rax), %%ymm2 \n\t" + " \n\t" + " \n\t" + " \n\t" + "addq $1 * 24 * 4, %%rax \n\t" // a += 1*24 (unroll x mr) + "addq $1 * 4 * 4, %%rbx \n\t" // b += 1*4 (unroll x nr) + " \n\t" + " \n\t" + "decq %%rsi \n\t" // i -= 1; + "jne .SLOOPKLEFT \n\t" // iterate again if i != 0. + " \n\t" + " \n\t" + " \n\t" + ".SPOSTACCUM: \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "movq %4, %%rax \n\t" // load address of alpha + "movq %5, %%rbx \n\t" // load address of beta + "vbroadcastss (%%rax), %%ymm0 \n\t" // load alpha and duplicate + "vbroadcastss (%%rbx), %%ymm3 \n\t" // load beta and duplicate + " \n\t" + "vmulps %%ymm0, %%ymm4, %%ymm4 \n\t" // scale by alpha + "vmulps %%ymm0, %%ymm5, %%ymm5 \n\t" + "vmulps %%ymm0, %%ymm6, %%ymm6 \n\t" + "vmulps %%ymm0, %%ymm7, %%ymm7 \n\t" + "vmulps %%ymm0, %%ymm8, %%ymm8 \n\t" + "vmulps %%ymm0, %%ymm9, %%ymm9 \n\t" + "vmulps %%ymm0, %%ymm10, %%ymm10 \n\t" + "vmulps %%ymm0, %%ymm11, %%ymm11 \n\t" + "vmulps %%ymm0, %%ymm12, %%ymm12 \n\t" + "vmulps %%ymm0, %%ymm13, %%ymm13 \n\t" + "vmulps %%ymm0, %%ymm14, %%ymm14 \n\t" + "vmulps %%ymm0, %%ymm15, %%ymm15 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "movq %7, %%rsi \n\t" // load rs_c + "leaq (,%%rsi,4), %%rsi \n\t" // rsi = rs_c * sizeof(float) + " \n\t" + "leaq (%%rcx,%%rsi,8), %%rdx \n\t" // rdx = c + 8*rs_c; + "leaq (%%rdx,%%rsi,8), %%r12 \n\t" // r12 = c + 16*rs_c; + " \n\t" + "leaq (%%rsi,%%rsi,2), %%r13 \n\t" // r13 = 3*rs_c; + "leaq (%%rsi,%%rsi,4), %%r15 \n\t" // r15 = 5*rs_c; + "leaq (%%r13,%%rsi,4), %%r10 \n\t" // r10 = 7*rs_c; + " \n\t" + " \n\t" + " \n\t" + " \n\t" // determine if + " \n\t" // c % 32 == 0, AND + " \n\t" // 8*cs_c % 32 == 0, AND + " \n\t" // rs_c == 1 + " \n\t" // ie: aligned, ldim aligned, and + " \n\t" // column-stored + " \n\t" + "cmpq $4, %%rsi \n\t" // set ZF if (4*rs_c) == 4. + "sete %%bl \n\t" // bl = ( ZF == 1 ? 1 : 0 ); + "testq $31, %%rcx \n\t" // set ZF if c & 32 is zero. + "setz %%bh \n\t" // bh = ( ZF == 0 ? 1 : 0 ); + "testq $31, %%rdi \n\t" // set ZF if (4*cs_c) & 32 is zero. + "setz %%al \n\t" // al = ( ZF == 0 ? 1 : 0 ); + " \n\t" // and(bl,bh) followed by + " \n\t" // and(bh,al) will reveal result + " \n\t" + " \n\t" // now avoid loading C if beta == 0 + " \n\t" + "vxorps %%ymm0, %%ymm0, %%ymm0 \n\t" // set ymm0 to zero. + "vucomiss %%xmm0, %%xmm3 \n\t" // set ZF if beta == 0. + "je .SBETAZERO \n\t" // if ZF = 1, jump to beta == 0 case + " \n\t" + " \n\t" + " \n\t" // check if aligned/column-stored + "andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. + "andb %%bh, %%al \n\t" // set ZF if bh & al == 1. + "jne .SCOLSTORED \n\t" // jump to column storage case + " \n\t" + " \n\t" + " \n\t" + ".SGENSTORED: \n\t" + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm4, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm7, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm10, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm13, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + //"addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "movq %%rdx, %%rcx \n\t" // rcx = c + 8*rs_c + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm5, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm8, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm11, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm14, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + //"addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "movq %%r12, %%rcx \n\t" // rcx = c + 16*rs_c + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm6, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm9, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm12, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm15, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + //"addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "jmp .SDONE \n\t" // jump to end. + " \n\t" + " \n\t" + " \n\t" + ".SCOLSTORED: \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213ps %%ymm4, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213ps %%ymm5, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovaps (%%r12), %%ymm2 \n\t" + "vfmadd213ps %%ymm6, %%ymm3, %%ymm2 \n\t" + "vmovaps %%ymm2, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213ps %%ymm7, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213ps %%ymm8, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovaps (%%r12), %%ymm2 \n\t" + "vfmadd213ps %%ymm9, %%ymm3, %%ymm2 \n\t" + "vmovaps %%ymm2, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213ps %%ymm10, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213ps %%ymm11, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovaps (%%r12), %%ymm2 \n\t" + "vfmadd213ps %%ymm12, %%ymm3, %%ymm2 \n\t" + "vmovaps %%ymm2, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213ps %%ymm13, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213ps %%ymm14, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovaps (%%r12), %%ymm2 \n\t" + "vfmadd213ps %%ymm15, %%ymm3, %%ymm2 \n\t" + "vmovaps %%ymm2, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "jmp .SDONE \n\t" // jump to end. + " \n\t" + " \n\t" + " \n\t" + ".SBETAZERO: \n\t" + " \n\t" // check if aligned/column-stored + "andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. + "andb %%bh, %%al \n\t" // set ZF if bh & al == 1. + "jne .SCOLSTORBZ \n\t" // jump to column storage case + " \n\t" + " \n\t" + " \n\t" + ".SGENSTORBZ: \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm4, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm7, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm10, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm13, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + //"addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "movq %%rdx, %%rcx \n\t" // rcx = c + 8*rs_c + " \n\t" + " \n\t" + "vmovaps %%ymm5, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm8, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm11, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm14, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + //"addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "movq %%r12, %%rcx \n\t" // rcx = c + 16*rs_c + " \n\t" + " \n\t" + "vmovaps %%ymm6, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm9, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm12, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm15, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + //"addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "jmp .SDONE \n\t" // jump to end. + " \n\t" + " \n\t" + " \n\t" + ".SCOLSTORBZ: \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm4, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm5, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovaps %%ymm6, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + "vmovaps %%ymm7, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm8, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovaps %%ymm9, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + "vmovaps %%ymm10, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm11, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovaps %%ymm12, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + "vmovaps %%ymm13, (%%rcx) \n\t" + //"addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm14, (%%rdx) \n\t" + //"addq %%rdi, %%rdx \n\t" + "vmovaps %%ymm15, (%%r12) \n\t" + //"addq %%rdi, %%r12 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + ".SDONE: \n\t" + " \n\t" + + : // output operands (none) + : // input operands + "m" (k_iter), // 0 + "m" (k_left), // 1 + "m" (a), // 2 + "m" (b), // 3 + "m" (alpha), // 4 + "m" (beta), // 5 + "m" (c), // 6 + "m" (rs_c), // 7 + "m" (cs_c)/*, // 8 + "m" (b_next), // 9 + "m" (a_next)*/ // 10 + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm6", "xmm7", + "xmm8", "xmm9", "xmm10", "xmm11", + "xmm12", "xmm13", "xmm14", "xmm15", + "memory" + ); +} + +#define DGEMM_INPUT_GS_BETA_NZ \ + "vmovlpd (%%rcx ), %%xmm0, %%xmm0 \n\t" \ + "vmovhpd (%%rcx,%%rsi,1), %%xmm0, %%xmm0 \n\t" \ + "vmovlpd (%%rcx,%%rsi,2), %%xmm1, %%xmm1 \n\t" \ + "vmovhpd (%%rcx,%%r13 ), %%xmm1, %%xmm1 \n\t" \ + "vperm2f128 $0x20, %%ymm1, %%ymm0, %%ymm0 \n\t" /*\ + "vmovlps (%%rcx,%%rsi,4), %%xmm2, %%xmm2 \n\t" \ + "vmovhps (%%rcx,%%r15 ), %%xmm2, %%xmm2 \n\t" \ + "vmovlps (%%rcx,%%r13,2), %%xmm1, %%xmm1 \n\t" \ + "vmovhps (%%rcx,%%r10 ), %%xmm1, %%xmm1 \n\t" \ + "vperm2f128 $0x20, %%ymm1, %%ymm2, %%ymm2 \n\t"*/ + +#define DGEMM_OUTPUT_GS_BETA_NZ \ + "vextractf128 $1, %%ymm0, %%xmm1 \n\t" \ + "vmovlpd %%xmm0, (%%rcx ) \n\t" \ + "vmovhpd %%xmm0, (%%rcx,%%rsi ) \n\t" \ + "vmovlpd %%xmm1, (%%rcx,%%rsi,2) \n\t" \ + "vmovhpd %%xmm1, (%%rcx,%%r13 ) \n\t" /*\ + "vextractf128 $1, %%ymm2, %%xmm1 \n\t" \ + "vmovlpd %%xmm2, (%%rcx,%%rsi,4) \n\t" \ + "vmovhpd %%xmm2, (%%rcx,%%r15 ) \n\t" \ + "vmovlpd %%xmm1, (%%rcx,%%r13,2) \n\t" \ + "vmovhpd %%xmm1, (%%rcx,%%r10 ) \n\t"*/ + +void bli_dgemm_asm_12x4( + dim_t k, + double* restrict alpha, + double* restrict a, + double* restrict b, + double* restrict beta, + double* restrict c, inc_t rs_c, inc_t cs_c, + auxinfo_t* data + ) +{ + //void* a_next = bli_auxinfo_next_a( data ); + //void* b_next = bli_auxinfo_next_b( data ); + + dim_t k_iter = k / 4; + dim_t k_left = k % 4; + + __asm__ volatile + ( + " \n\t" + " \n\t" + "movq %2, %%rax \n\t" // load address of a. + "movq %3, %%rbx \n\t" // load address of b. + //"movq %9, %%r15 \n\t" // load address of b_next. + " \n\t" + " \n\t" // initialize loop by pre-loading + "vmovapd 0 * 32(%%rax), %%ymm0 \n\t" + "vmovapd 1 * 32(%%rax), %%ymm1 \n\t" + "vmovapd 2 * 32(%%rax), %%ymm2 \n\t" + " \n\t" + "movq %6, %%rcx \n\t" // load address of c + "movq %8, %%rdi \n\t" // load cs_c + "leaq (,%%rdi,8), %%rdi \n\t" // cs_c *= sizeof(double) + " \n\t" + "leaq (%%rdi,%%rdi,2), %%r13 \n\t" // r13 = 3*cs_c; + "prefetcht0 7 * 8(%%rcx) \n\t" // prefetch c + 0*cs_c + "prefetcht0 7 * 8(%%rcx,%%rdi) \n\t" // prefetch c + 1*cs_c + "prefetcht0 7 * 8(%%rcx,%%rdi,2) \n\t" // prefetch c + 2*cs_c + "prefetcht0 7 * 8(%%rcx,%%r13) \n\t" // prefetch c + 3*cs_c + " \n\t" + "vxorpd %%ymm4, %%ymm4, %%ymm4 \n\t" + "vxorpd %%ymm5, %%ymm5, %%ymm5 \n\t" + "vxorpd %%ymm6, %%ymm6, %%ymm6 \n\t" + "vxorpd %%ymm7, %%ymm7, %%ymm7 \n\t" + "vxorpd %%ymm8, %%ymm8, %%ymm8 \n\t" + "vxorpd %%ymm9, %%ymm9, %%ymm9 \n\t" + "vxorpd %%ymm10, %%ymm10, %%ymm10 \n\t" + "vxorpd %%ymm11, %%ymm11, %%ymm11 \n\t" + "vxorpd %%ymm12, %%ymm12, %%ymm12 \n\t" + "vxorpd %%ymm13, %%ymm13, %%ymm13 \n\t" + "vxorpd %%ymm14, %%ymm14, %%ymm14 \n\t" + "vxorpd %%ymm15, %%ymm15, %%ymm15 \n\t" + " \n\t" + " \n\t" + " \n\t" + "movq %0, %%rsi \n\t" // i = k_iter; + "testq %%rsi, %%rsi \n\t" // check i via logical AND. + "je .DCONSIDKLEFT \n\t" // if i == 0, jump to code that + " \n\t" // contains the k_left loop. + " \n\t" + " \n\t" + ".DLOOPKITER: \n\t" // MAIN LOOP + " \n\t" + " \n\t" + " \n\t" // iteration 0 + "prefetcht0 16 * 32(%%rax) \n\t" + " \n\t" + "vbroadcastsd 0 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm4 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm5 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm6 \n\t" + " \n\t" + "vbroadcastsd 1 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm7 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm8 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm9 \n\t" + " \n\t" + "vbroadcastsd 2 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm11 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm12 \n\t" + " \n\t" + "vbroadcastsd 3 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm13 \n\t" + "vmovapd 3 * 32(%%rax), %%ymm0 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm14 \n\t" + "vmovapd 4 * 32(%%rax), %%ymm1 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm15 \n\t" + "vmovapd 5 * 32(%%rax), %%ymm2 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" // iteration 1 + "vbroadcastsd 4 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm4 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm5 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm6 \n\t" + " \n\t" + "vbroadcastsd 5 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm7 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm8 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm9 \n\t" + " \n\t" + "vbroadcastsd 6 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm11 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm12 \n\t" + " \n\t" + "vbroadcastsd 7 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm13 \n\t" + "vmovapd 6 * 32(%%rax), %%ymm0 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm14 \n\t" + "vmovapd 7 * 32(%%rax), %%ymm1 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm15 \n\t" + "vmovapd 8 * 32(%%rax), %%ymm2 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" // iteration 2 + "prefetcht0 22 * 32(%%rax) \n\t" + " \n\t" + "vbroadcastsd 8 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm4 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm5 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm6 \n\t" + " \n\t" + "vbroadcastsd 9 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm7 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm8 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm9 \n\t" + " \n\t" + "vbroadcastsd 10 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm11 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm12 \n\t" + " \n\t" + "vbroadcastsd 11 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm13 \n\t" + "vmovapd 9 * 32(%%rax), %%ymm0 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm14 \n\t" + "vmovapd 10 * 32(%%rax), %%ymm1 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm15 \n\t" + "vmovapd 11 * 32(%%rax), %%ymm2 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" // iteration 3 + "vbroadcastsd 12 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm4 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm5 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm6 \n\t" + " \n\t" + "vbroadcastsd 13 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm7 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm8 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm9 \n\t" + " \n\t" + "vbroadcastsd 14 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm11 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm12 \n\t" + " \n\t" + "vbroadcastsd 15 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm13 \n\t" + "vmovapd 12 * 32(%%rax), %%ymm0 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm14 \n\t" + "vmovapd 13 * 32(%%rax), %%ymm1 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm15 \n\t" + "vmovapd 14 * 32(%%rax), %%ymm2 \n\t" + " \n\t" + " \n\t" + " \n\t" + "addq $4 * 12 * 8, %%rax \n\t" // a += 4*12 (unroll x mr) + "addq $4 * 4 * 8, %%rbx \n\t" // b += 4*4 (unroll x nr) + " \n\t" + " \n\t" + "decq %%rsi \n\t" // i -= 1; + "jne .DLOOPKITER \n\t" // iterate again if i != 0. + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + ".DCONSIDKLEFT: \n\t" + " \n\t" + "movq %1, %%rsi \n\t" // i = k_left; + "testq %%rsi, %%rsi \n\t" // check i via logical AND. + "je .DPOSTACCUM \n\t" // if i == 0, we're done; jump to end. + " \n\t" // else, we prepare to enter k_left loop. + " \n\t" + " \n\t" + ".DLOOPKLEFT: \n\t" // EDGE LOOP + " \n\t" + "prefetcht0 16 * 32(%%rax) \n\t" + " \n\t" + "vbroadcastsd 0 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm4 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm5 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm6 \n\t" + " \n\t" + "vbroadcastsd 1 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm7 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm8 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm9 \n\t" + " \n\t" + "vbroadcastsd 2 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm11 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm12 \n\t" + " \n\t" + "vbroadcastsd 3 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm13 \n\t" + "vmovapd 3 * 32(%%rax), %%ymm0 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm14 \n\t" + "vmovapd 4 * 32(%%rax), %%ymm1 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm15 \n\t" + "vmovapd 5 * 32(%%rax), %%ymm2 \n\t" + " \n\t" + " \n\t" + " \n\t" + "addq $1 * 12 * 8, %%rax \n\t" // a += 1*12 (unroll x mr) + "addq $1 * 4 * 8, %%rbx \n\t" // b += 1*4 (unroll x nr) + " \n\t" + " \n\t" + "decq %%rsi \n\t" // i -= 1; + "jne .DLOOPKLEFT \n\t" // iterate again if i != 0. + " \n\t" + " \n\t" + " \n\t" + ".DPOSTACCUM: \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "movq %4, %%rax \n\t" // load address of alpha + "movq %5, %%rbx \n\t" // load address of beta + "vbroadcastsd (%%rax), %%ymm0 \n\t" // load alpha and duplicate + "vbroadcastsd (%%rbx), %%ymm3 \n\t" // load beta and duplicate + " \n\t" + "vmulpd %%ymm0, %%ymm4, %%ymm4 \n\t" // scale by alpha + "vmulpd %%ymm0, %%ymm5, %%ymm5 \n\t" + "vmulpd %%ymm0, %%ymm6, %%ymm6 \n\t" + "vmulpd %%ymm0, %%ymm7, %%ymm7 \n\t" + "vmulpd %%ymm0, %%ymm8, %%ymm8 \n\t" + "vmulpd %%ymm0, %%ymm9, %%ymm9 \n\t" + "vmulpd %%ymm0, %%ymm10, %%ymm10 \n\t" + "vmulpd %%ymm0, %%ymm11, %%ymm11 \n\t" + "vmulpd %%ymm0, %%ymm12, %%ymm12 \n\t" + "vmulpd %%ymm0, %%ymm13, %%ymm13 \n\t" + "vmulpd %%ymm0, %%ymm14, %%ymm14 \n\t" + "vmulpd %%ymm0, %%ymm15, %%ymm15 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "movq %7, %%rsi \n\t" // load rs_c + "leaq (,%%rsi,8), %%rsi \n\t" // rsi = rs_c * sizeof(double) + " \n\t" + "leaq (%%rcx,%%rsi,4), %%rdx \n\t" // rdx = c + 4*rs_c; + "leaq (%%rcx,%%rsi,8), %%r12 \n\t" // r12 = c + 8*rs_c; + " \n\t" + "leaq (%%rsi,%%rsi,2), %%r13 \n\t" // r13 = 3*rs_c; + //"leaq (%%rsi,%%rsi,4), %%r15 \n\t" // r15 = 5*rs_c; + //"leaq (%%r13,%%rsi,4), %%r10 \n\t" // r10 = 7*rs_c; + " \n\t" + " \n\t" + " \n\t" + " \n\t" // determine if + " \n\t" // c % 32 == 0, AND + " \n\t" // 8*cs_c % 32 == 0, AND + " \n\t" // rs_c == 1 + " \n\t" // ie: aligned, ldim aligned, and + " \n\t" // column-stored + " \n\t" + "cmpq $8, %%rsi \n\t" // set ZF if (8*rs_c) == 8. + "sete %%bl \n\t" // bl = ( ZF == 1 ? 1 : 0 ); + "testq $31, %%rcx \n\t" // set ZF if c & 32 is zero. + "setz %%bh \n\t" // bh = ( ZF == 0 ? 1 : 0 ); + "testq $31, %%rdi \n\t" // set ZF if (8*cs_c) & 32 is zero. + "setz %%al \n\t" // al = ( ZF == 0 ? 1 : 0 ); + " \n\t" // and(bl,bh) followed by + " \n\t" // and(bh,al) will reveal result + " \n\t" + " \n\t" // now avoid loading C if beta == 0 + " \n\t" + "vxorpd %%ymm0, %%ymm0, %%ymm0 \n\t" // set ymm0 to zero. + "vucomisd %%xmm0, %%xmm3 \n\t" // set ZF if beta == 0. + "je .DBETAZERO \n\t" // if ZF = 1, jump to beta == 0 case + " \n\t" + " \n\t" + " \n\t" // check if aligned/column-stored + "andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. + "andb %%bh, %%al \n\t" // set ZF if bh & al == 1. + "jne .DCOLSTORED \n\t" // jump to column storage case + " \n\t" + " \n\t" + " \n\t" + ".DGENSTORED: \n\t" + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm4, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm7, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm10, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm13, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + //"addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "movq %%rdx, %%rcx \n\t" // rcx = c + 4*rs_c + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm5, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm8, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm11, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm14, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + //"addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "movq %%r12, %%rcx \n\t" // rcx = c + 8*rs_c + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm6, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm9, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm12, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm15, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + //"addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "jmp .DDONE \n\t" // jump to end. + " \n\t" + " \n\t" + " \n\t" + ".DCOLSTORED: \n\t" + " \n\t" + " \n\t" + "vmovapd (%%rcx), %%ymm0 \n\t" + "vfmadd213pd %%ymm4, %%ymm3, %%ymm0 \n\t" + "vmovapd %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovapd (%%rdx), %%ymm1 \n\t" + "vfmadd213pd %%ymm5, %%ymm3, %%ymm1 \n\t" + "vmovapd %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovapd (%%r12), %%ymm2 \n\t" + "vfmadd213pd %%ymm6, %%ymm3, %%ymm2 \n\t" + "vmovapd %%ymm2, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + " \n\t" + "vmovapd (%%rcx), %%ymm0 \n\t" + "vfmadd213pd %%ymm7, %%ymm3, %%ymm0 \n\t" + "vmovapd %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovapd (%%rdx), %%ymm1 \n\t" + "vfmadd213pd %%ymm8, %%ymm3, %%ymm1 \n\t" + "vmovapd %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovapd (%%r12), %%ymm2 \n\t" + "vfmadd213pd %%ymm9, %%ymm3, %%ymm2 \n\t" + "vmovapd %%ymm2, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + " \n\t" + "vmovapd (%%rcx), %%ymm0 \n\t" + "vfmadd213pd %%ymm10, %%ymm3, %%ymm0 \n\t" + "vmovapd %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovapd (%%rdx), %%ymm1 \n\t" + "vfmadd213pd %%ymm11, %%ymm3, %%ymm1 \n\t" + "vmovapd %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovapd (%%r12), %%ymm2 \n\t" + "vfmadd213pd %%ymm12, %%ymm3, %%ymm2 \n\t" + "vmovapd %%ymm2, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + " \n\t" + "vmovapd (%%rcx), %%ymm0 \n\t" + "vfmadd213pd %%ymm13, %%ymm3, %%ymm0 \n\t" + "vmovapd %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovapd (%%rdx), %%ymm1 \n\t" + "vfmadd213pd %%ymm14, %%ymm3, %%ymm1 \n\t" + "vmovapd %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovapd (%%r12), %%ymm2 \n\t" + "vfmadd213pd %%ymm15, %%ymm3, %%ymm2 \n\t" + "vmovapd %%ymm2, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "jmp .DDONE \n\t" // jump to end. + " \n\t" + " \n\t" + " \n\t" + ".DBETAZERO: \n\t" + " \n\t" // check if aligned/column-stored + "andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. + "andb %%bh, %%al \n\t" // set ZF if bh & al == 1. + "jne .DCOLSTORBZ \n\t" // jump to column storage case + " \n\t" + " \n\t" + " \n\t" + ".DGENSTORBZ: \n\t" + " \n\t" + " \n\t" + "vmovapd %%ymm4, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovapd %%ymm7, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovapd %%ymm10, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovapd %%ymm13, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + //"addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "movq %%rdx, %%rcx \n\t" // rcx = c + 4*rs_c + " \n\t" + " \n\t" + "vmovapd %%ymm5, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovapd %%ymm8, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovapd %%ymm11, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovapd %%ymm14, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + //"addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "movq %%r12, %%rcx \n\t" // rcx = c + 8*rs_c + " \n\t" + " \n\t" + "vmovapd %%ymm6, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovapd %%ymm9, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovapd %%ymm12, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovapd %%ymm15, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + //"addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "jmp .DDONE \n\t" // jump to end. + " \n\t" + " \n\t" + " \n\t" + ".DCOLSTORBZ: \n\t" + " \n\t" + " \n\t" + "vmovapd %%ymm4, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovapd %%ymm5, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovapd %%ymm6, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + "vmovapd %%ymm7, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovapd %%ymm8, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovapd %%ymm9, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + "vmovapd %%ymm10, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovapd %%ymm11, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovapd %%ymm12, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + "vmovapd %%ymm13, (%%rcx) \n\t" + //"addq %%rdi, %%rcx \n\t" + "vmovapd %%ymm14, (%%rdx) \n\t" + //"addq %%rdi, %%rdx \n\t" + "vmovapd %%ymm15, (%%r12) \n\t" + //"addq %%rdi, %%r12 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + ".DDONE: \n\t" + " \n\t" + + : // output operands (none) + : // input operands + "m" (k_iter), // 0 + "m" (k_left), // 1 + "m" (a), // 2 + "m" (b), // 3 + "m" (alpha), // 4 + "m" (beta), // 5 + "m" (c), // 6 + "m" (rs_c), // 7 + "m" (cs_c)/*, // 8 + "m" (b_next), // 9 + "m" (a_next)*/ // 10 + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm6", "xmm7", + "xmm8", "xmm9", "xmm10", "xmm11", + "xmm12", "xmm13", "xmm14", "xmm15", + "memory" + ); +} + +#if 0 + +void bli_cgemm_asm_( + dim_t k, + scomplex* restrict alpha, + scomplex* restrict a, + scomplex* restrict b, + scomplex* restrict beta, + scomplex* restrict c, inc_t rs_c, inc_t cs_c, + auxinfo_t* data + ) +{ + //void* a_next = bli_auxinfo_next_a( data ); + //void* b_next = bli_auxinfo_next_b( data ); + + //dim_t k_iter = k / 4; + //dim_t k_left = k % 4; + +} + + + +void bli_zgemm_asm_( + dim_t k, + dcomplex* restrict alpha, + dcomplex* restrict a, + dcomplex* restrict b, + dcomplex* restrict beta, + dcomplex* restrict c, inc_t rs_c, inc_t cs_c, + auxinfo_t* data + ) +{ + //void* a_next = bli_auxinfo_next_a( data ); + //void* b_next = bli_auxinfo_next_b( data ); + + //dim_t k_iter = k / 4; + //dim_t k_left = k % 4; + +} + +#endif diff --git a/kernels/x86_64/avx2/3/bli_gemm_asm_d8x6.c b/kernels/x86_64/avx2/3/bli_gemm_asm_d8x6.c new file mode 100644 index 0000000000..9c11950fc7 --- /dev/null +++ b/kernels/x86_64/avx2/3/bli_gemm_asm_d8x6.c @@ -0,0 +1,1326 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + + +#define SGEMM_INPUT_GS_BETA_NZ \ + "vmovlps (%%rcx ), %%xmm0, %%xmm0 \n\t" \ + "vmovhps (%%rcx,%%rsi,1), %%xmm0, %%xmm0 \n\t" \ + "vmovlps (%%rcx,%%rsi,2), %%xmm1, %%xmm1 \n\t" \ + "vmovhps (%%rcx,%%r13 ), %%xmm1, %%xmm1 \n\t" \ + "vshufps $0x88, %%xmm1, %%xmm0, %%xmm0 \n\t" \ + "vmovlps (%%rcx,%%rsi,4), %%xmm2, %%xmm2 \n\t" \ + "vmovhps (%%rcx,%%r15 ), %%xmm2, %%xmm2 \n\t" \ + "vmovlps (%%rcx,%%r13,2), %%xmm1, %%xmm1 \n\t" \ + "vmovhps (%%rcx,%%r10 ), %%xmm1, %%xmm1 \n\t" \ + "vshufps $0x88, %%xmm1, %%xmm2, %%xmm2 \n\t" \ + "vperm2f128 $0x20, %%ymm2, %%ymm0, %%ymm0 \n\t" + +#define SGEMM_OUTPUT_GS_BETA_NZ \ + "vextractf128 $1, %%ymm0, %%xmm2 \n\t" \ + "vmovss %%xmm0, (%%rcx ) \n\t" \ + "vpermilps $0x39, %%xmm0, %%xmm1 \n\t" \ + "vmovss %%xmm1, (%%rcx,%%rsi,1) \n\t" \ + "vpermilps $0x39, %%xmm1, %%xmm0 \n\t" \ + "vmovss %%xmm0, (%%rcx,%%rsi,2) \n\t" \ + "vpermilps $0x39, %%xmm0, %%xmm1 \n\t" \ + "vmovss %%xmm1, (%%rcx,%%r13 ) \n\t" \ + "vmovss %%xmm2, (%%rcx,%%rsi,4) \n\t" \ + "vpermilps $0x39, %%xmm2, %%xmm1 \n\t" \ + "vmovss %%xmm1, (%%rcx,%%r15 ) \n\t" \ + "vpermilps $0x39, %%xmm1, %%xmm2 \n\t" \ + "vmovss %%xmm2, (%%rcx,%%r13,2) \n\t" \ + "vpermilps $0x39, %%xmm2, %%xmm1 \n\t" \ + "vmovss %%xmm1, (%%rcx,%%r10 ) \n\t" + +void bli_sgemm_asm_16x6( + dim_t k, + float* restrict alpha, + float* restrict a, + float* restrict b, + float* restrict beta, + float* restrict c, inc_t rs_c, inc_t cs_c, + auxinfo_t* data + ) +{ + //void* a_next = bli_auxinfo_next_a( data ); + //void* b_next = bli_auxinfo_next_b( data ); + + dim_t k_iter = k / 4; + dim_t k_left = k % 4; + + __asm__ volatile + ( + " \n\t" + " \n\t" + "movq %2, %%rax \n\t" // load address of a. + "movq %3, %%rbx \n\t" // load address of b. + //"movq %9, %%r15 \n\t" // load address of b_next. + " \n\t" + "addq $32 * 4, %%rax \n\t" + " \n\t" // initialize loop by pre-loading + "vmovaps -4 * 32(%%rax), %%ymm0 \n\t" + "vmovaps -3 * 32(%%rax), %%ymm1 \n\t" + " \n\t" + "movq %6, %%rcx \n\t" // load address of c + "movq %8, %%rdi \n\t" // load cs_c + "leaq (,%%rdi,4), %%rdi \n\t" // cs_c *= sizeof(float) + " \n\t" + "leaq (%%rdi,%%rdi,2), %%r13 \n\t" // r13 = 3*cs_c; + "leaq (%%rcx,%%r13,1), %%rdx \n\t" // rdx = c + 3*cs_c; + "prefetcht0 7 * 8(%%rcx) \n\t" // prefetch c + 0*cs_c + "prefetcht0 7 * 8(%%rcx,%%rdi) \n\t" // prefetch c + 1*cs_c + "prefetcht0 7 * 8(%%rcx,%%rdi,2) \n\t" // prefetch c + 2*cs_c + "prefetcht0 7 * 8(%%rdx) \n\t" // prefetch c + 3*cs_c + "prefetcht0 7 * 8(%%rdx,%%rdi) \n\t" // prefetch c + 4*cs_c + "prefetcht0 7 * 8(%%rdx,%%rdi,2) \n\t" // prefetch c + 5*cs_c + " \n\t" + "vxorps %%ymm4, %%ymm4, %%ymm4 \n\t" + "vxorps %%ymm5, %%ymm5, %%ymm5 \n\t" + "vxorps %%ymm6, %%ymm6, %%ymm6 \n\t" + "vxorps %%ymm7, %%ymm7, %%ymm7 \n\t" + "vxorps %%ymm8, %%ymm8, %%ymm8 \n\t" + "vxorps %%ymm9, %%ymm9, %%ymm9 \n\t" + "vxorps %%ymm10, %%ymm10, %%ymm10 \n\t" + "vxorps %%ymm11, %%ymm11, %%ymm11 \n\t" + "vxorps %%ymm12, %%ymm12, %%ymm12 \n\t" + "vxorps %%ymm13, %%ymm13, %%ymm13 \n\t" + "vxorps %%ymm14, %%ymm14, %%ymm14 \n\t" + "vxorps %%ymm15, %%ymm15, %%ymm15 \n\t" + " \n\t" + " \n\t" + " \n\t" + "movq %0, %%rsi \n\t" // i = k_iter; + "testq %%rsi, %%rsi \n\t" // check i via logical AND. + "je .SCONSIDKLEFT \n\t" // if i == 0, jump to code that + " \n\t" // contains the k_left loop. + " \n\t" + " \n\t" + ".SLOOPKITER: \n\t" // MAIN LOOP + " \n\t" + " \n\t" + " \n\t" // iteration 0 + "prefetcht0 16 * 32(%%rax) \n\t" + " \n\t" + "vbroadcastss 0 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 1 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm4 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm5 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm6 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm7 \n\t" + " \n\t" + "vbroadcastss 2 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 3 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm8 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm9 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm11 \n\t" + " \n\t" + "vbroadcastss 4 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 5 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm12 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm13 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm14 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm15 \n\t" + " \n\t" + "vmovaps -2 * 32(%%rax), %%ymm0 \n\t" + "vmovaps -1 * 32(%%rax), %%ymm1 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" // iteration 1 + "vbroadcastss 6 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 7 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm4 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm5 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm6 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm7 \n\t" + " \n\t" + "vbroadcastss 8 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 9 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm8 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm9 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm11 \n\t" + " \n\t" + "vbroadcastss 10 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 11 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm12 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm13 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm14 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm15 \n\t" + " \n\t" + "vmovaps 0 * 32(%%rax), %%ymm0 \n\t" + "vmovaps 1 * 32(%%rax), %%ymm1 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" // iteration 2 + "prefetcht0 20 * 32(%%rax) \n\t" + " \n\t" + "vbroadcastss 12 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 13 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm4 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm5 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm6 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm7 \n\t" + " \n\t" + "vbroadcastss 14 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 15 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm8 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm9 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm11 \n\t" + " \n\t" + "vbroadcastss 16 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 17 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm12 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm13 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm14 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm15 \n\t" + " \n\t" + "vmovaps 2 * 32(%%rax), %%ymm0 \n\t" + "vmovaps 3 * 32(%%rax), %%ymm1 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" // iteration 3 + "vbroadcastss 18 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 19 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm4 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm5 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm6 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm7 \n\t" + " \n\t" + "vbroadcastss 20 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 21 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm8 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm9 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm11 \n\t" + " \n\t" + "vbroadcastss 22 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 23 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm12 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm13 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm14 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm15 \n\t" + " \n\t" + "addq $4 * 16 * 4, %%rax \n\t" // a += 4*16 (unroll x mr) + "addq $4 * 6 * 4, %%rbx \n\t" // b += 4*6 (unroll x nr) + " \n\t" + "vmovaps -4 * 32(%%rax), %%ymm0 \n\t" + "vmovaps -3 * 32(%%rax), %%ymm1 \n\t" + " \n\t" + " \n\t" + "decq %%rsi \n\t" // i -= 1; + "jne .SLOOPKITER \n\t" // iterate again if i != 0. + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + ".SCONSIDKLEFT: \n\t" + " \n\t" + "movq %1, %%rsi \n\t" // i = k_left; + "testq %%rsi, %%rsi \n\t" // check i via logical AND. + "je .SPOSTACCUM \n\t" // if i == 0, we're done; jump to end. + " \n\t" // else, we prepare to enter k_left loop. + " \n\t" + " \n\t" + ".SLOOPKLEFT: \n\t" // EDGE LOOP + " \n\t" + "prefetcht0 16 * 32(%%rax) \n\t" + " \n\t" + "vbroadcastss 0 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 1 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm4 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm5 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm6 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm7 \n\t" + " \n\t" + "vbroadcastss 2 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 3 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm8 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm9 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm11 \n\t" + " \n\t" + "vbroadcastss 4 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 5 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm12 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm13 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm14 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm15 \n\t" + " \n\t" + "addq $1 * 16 * 4, %%rax \n\t" // a += 1*16 (unroll x mr) + "addq $1 * 6 * 4, %%rbx \n\t" // b += 1*6 (unroll x nr) + " \n\t" + "vmovaps -4 * 32(%%rax), %%ymm0 \n\t" + "vmovaps -3 * 32(%%rax), %%ymm1 \n\t" + " \n\t" + " \n\t" + "decq %%rsi \n\t" // i -= 1; + "jne .SLOOPKLEFT \n\t" // iterate again if i != 0. + " \n\t" + " \n\t" + " \n\t" + ".SPOSTACCUM: \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "movq %4, %%rax \n\t" // load address of alpha + "movq %5, %%rbx \n\t" // load address of beta + "vbroadcastss (%%rax), %%ymm0 \n\t" // load alpha and duplicate + "vbroadcastss (%%rbx), %%ymm3 \n\t" // load beta and duplicate + " \n\t" + "vmulps %%ymm0, %%ymm4, %%ymm4 \n\t" // scale by alpha + "vmulps %%ymm0, %%ymm5, %%ymm5 \n\t" + "vmulps %%ymm0, %%ymm6, %%ymm6 \n\t" + "vmulps %%ymm0, %%ymm7, %%ymm7 \n\t" + "vmulps %%ymm0, %%ymm8, %%ymm8 \n\t" + "vmulps %%ymm0, %%ymm9, %%ymm9 \n\t" + "vmulps %%ymm0, %%ymm10, %%ymm10 \n\t" + "vmulps %%ymm0, %%ymm11, %%ymm11 \n\t" + "vmulps %%ymm0, %%ymm12, %%ymm12 \n\t" + "vmulps %%ymm0, %%ymm13, %%ymm13 \n\t" + "vmulps %%ymm0, %%ymm14, %%ymm14 \n\t" + "vmulps %%ymm0, %%ymm15, %%ymm15 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "movq %7, %%rsi \n\t" // load rs_c + "leaq (,%%rsi,4), %%rsi \n\t" // rsi = rs_c * sizeof(float) + " \n\t" + "leaq (%%rcx,%%rsi,8), %%rdx \n\t" // load address of c + 8*rs_c; + " \n\t" + "leaq (%%rsi,%%rsi,2), %%r13 \n\t" // r13 = 3*rs_c; + "leaq (%%rsi,%%rsi,4), %%r15 \n\t" // r15 = 5*rs_c; + "leaq (%%r13,%%rsi,4), %%r10 \n\t" // r10 = 7*rs_c; + " \n\t" + " \n\t" + " \n\t" + " \n\t" // determine if + " \n\t" // c % 32 == 0, AND + " \n\t" // 4*cs_c % 32 == 0, AND + " \n\t" // rs_c == 1 + " \n\t" // ie: aligned, ldim aligned, and + " \n\t" // column-stored + " \n\t" + "cmpq $4, %%rsi \n\t" // set ZF if (4*rs_c) == 4. + "sete %%bl \n\t" // bl = ( ZF == 1 ? 1 : 0 ); + "testq $31, %%rcx \n\t" // set ZF if c & 32 is zero. + "setz %%bh \n\t" // bh = ( ZF == 0 ? 1 : 0 ); + "testq $31, %%rdi \n\t" // set ZF if (4*cs_c) & 32 is zero. + "setz %%al \n\t" // al = ( ZF == 0 ? 1 : 0 ); + " \n\t" // and(bl,bh) followed by + " \n\t" // and(bh,al) will reveal result + " \n\t" + " \n\t" // now avoid loading C if beta == 0 + " \n\t" + "vxorps %%ymm0, %%ymm0, %%ymm0 \n\t" // set ymm0 to zero. + "vucomiss %%xmm0, %%xmm3 \n\t" // set ZF if beta == 0. + "je .SBETAZERO \n\t" // if ZF = 1, jump to beta == 0 case + " \n\t" + " \n\t" + " \n\t" // check if aligned/column-stored + "andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. + "andb %%bh, %%al \n\t" // set ZF if bh & al == 1. + "jne .SCOLSTORED \n\t" // jump to column storage case + " \n\t" + " \n\t" + " \n\t" + ".SGENSTORED: \n\t" + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm4, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm6, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm8, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm10, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm12, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm14, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + " \n\t" + " \n\t" + "movq %%rdx, %%rcx \n\t" // rcx = c + 8*rs_c + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm5, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm7, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm9, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm11, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm13, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm15, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + " \n\t" + " \n\t" + " \n\t" + "jmp .SDONE \n\t" // jump to end. + " \n\t" + " \n\t" + " \n\t" + ".SCOLSTORED: \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213ps %%ymm4, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213ps %%ymm5, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213ps %%ymm6, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213ps %%ymm7, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213ps %%ymm8, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213ps %%ymm9, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213ps %%ymm10, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213ps %%ymm11, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213ps %%ymm12, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213ps %%ymm13, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213ps %%ymm14, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213ps %%ymm15, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + " \n\t" + "jmp .SDONE \n\t" // jump to end. + " \n\t" + " \n\t" + " \n\t" + ".SBETAZERO: \n\t" + " \n\t" // check if aligned/column-stored + "andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. + "andb %%bh, %%al \n\t" // set ZF if bh & al == 1. + "jne .SCOLSTORBZ \n\t" // jump to column storage case + " \n\t" + " \n\t" + " \n\t" + ".SGENSTORBZ: \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm4, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm6, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm8, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm10, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm12, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm14, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + " \n\t" + " \n\t" + "movq %%rdx, %%rcx \n\t" // rcx = c + 8*rs_c + " \n\t" + " \n\t" + "vmovaps %%ymm5, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm7, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm9, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm11, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm13, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm15, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + " \n\t" + " \n\t" + " \n\t" + "jmp .SDONE \n\t" // jump to end. + " \n\t" + " \n\t" + " \n\t" + ".SCOLSTORBZ: \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm4, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm5, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + "vmovaps %%ymm6, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm7, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm8, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm9, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm10, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm11, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm12, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm13, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm14, (%%rcx) \n\t" + " \n\t" + "vmovaps %%ymm15, (%%rdx) \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + ".SDONE: \n\t" + " \n\t" + + : // output operands (none) + : // input operands + "m" (k_iter), // 0 + "m" (k_left), // 1 + "m" (a), // 2 + "m" (b), // 3 + "m" (alpha), // 4 + "m" (beta), // 5 + "m" (c), // 6 + "m" (rs_c), // 7 + "m" (cs_c)/*, // 8 + "m" (b_next), // 9 + "m" (a_next)*/ // 10 + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm6", "xmm7", + "xmm8", "xmm9", "xmm10", "xmm11", + "xmm12", "xmm13", "xmm14", "xmm15", + "memory" + ); +} + +#define DGEMM_INPUT_GS_BETA_NZ \ + "vmovlpd (%%rcx ), %%xmm0, %%xmm0 \n\t" \ + "vmovhpd (%%rcx,%%rsi,1), %%xmm0, %%xmm0 \n\t" \ + "vmovlpd (%%rcx,%%rsi,2), %%xmm1, %%xmm1 \n\t" \ + "vmovhpd (%%rcx,%%r13 ), %%xmm1, %%xmm1 \n\t" \ + "vperm2f128 $0x20, %%ymm1, %%ymm0, %%ymm0 \n\t" /*\ + "vmovlps (%%rcx,%%rsi,4), %%xmm2, %%xmm2 \n\t" \ + "vmovhps (%%rcx,%%r15 ), %%xmm2, %%xmm2 \n\t" \ + "vmovlps (%%rcx,%%r13,2), %%xmm1, %%xmm1 \n\t" \ + "vmovhps (%%rcx,%%r10 ), %%xmm1, %%xmm1 \n\t" \ + "vperm2f128 $0x20, %%ymm1, %%ymm2, %%ymm2 \n\t"*/ + +#define DGEMM_OUTPUT_GS_BETA_NZ \ + "vextractf128 $1, %%ymm0, %%xmm1 \n\t" \ + "vmovlpd %%xmm0, (%%rcx ) \n\t" \ + "vmovhpd %%xmm0, (%%rcx,%%rsi ) \n\t" \ + "vmovlpd %%xmm1, (%%rcx,%%rsi,2) \n\t" \ + "vmovhpd %%xmm1, (%%rcx,%%r13 ) \n\t" /*\ + "vextractf128 $1, %%ymm2, %%xmm1 \n\t" \ + "vmovlpd %%xmm2, (%%rcx,%%rsi,4) \n\t" \ + "vmovhpd %%xmm2, (%%rcx,%%r15 ) \n\t" \ + "vmovlpd %%xmm1, (%%rcx,%%r13,2) \n\t" \ + "vmovhpd %%xmm1, (%%rcx,%%r10 ) \n\t"*/ + +void bli_dgemm_asm_8x6( + dim_t k, + double* restrict alpha, + double* restrict a, + double* restrict b, + double* restrict beta, + double* restrict c, inc_t rs_c, inc_t cs_c, + auxinfo_t* data + ) +{ + //void* a_next = bli_auxinfo_next_a( data ); + //void* b_next = bli_auxinfo_next_b( data ); + + dim_t k_iter = k / 4; + dim_t k_left = k % 4; + + __asm__ volatile + ( + " \n\t" + " \n\t" + "movq %2, %%rax \n\t" // load address of a. + "movq %3, %%rbx \n\t" // load address of b. + //"movq %9, %%r15 \n\t" // load address of b_next. + " \n\t" + "addq $32 * 4, %%rax \n\t" + " \n\t" // initialize loop by pre-loading + "vmovaps -4 * 32(%%rax), %%ymm0 \n\t" + "vmovaps -3 * 32(%%rax), %%ymm1 \n\t" + " \n\t" + "movq %6, %%rcx \n\t" // load address of c + "movq %8, %%rdi \n\t" // load cs_c + "leaq (,%%rdi,8), %%rdi \n\t" // cs_c *= sizeof(double) + " \n\t" + "leaq (%%rdi,%%rdi,2), %%r13 \n\t" // r13 = 3*cs_c; + "leaq (%%rcx,%%r13,1), %%rdx \n\t" // rdx = c + 3*cs_c; + "prefetcht0 7 * 8(%%rcx) \n\t" // prefetch c + 0*cs_c + "prefetcht0 7 * 8(%%rcx,%%rdi) \n\t" // prefetch c + 1*cs_c + "prefetcht0 7 * 8(%%rcx,%%rdi,2) \n\t" // prefetch c + 2*cs_c + "prefetcht0 7 * 8(%%rdx) \n\t" // prefetch c + 3*cs_c + "prefetcht0 7 * 8(%%rdx,%%rdi) \n\t" // prefetch c + 4*cs_c + "prefetcht0 7 * 8(%%rdx,%%rdi,2) \n\t" // prefetch c + 5*cs_c + " \n\t" + "vxorpd %%ymm4, %%ymm4, %%ymm4 \n\t" + "vxorpd %%ymm5, %%ymm5, %%ymm5 \n\t" + "vxorpd %%ymm6, %%ymm6, %%ymm6 \n\t" + "vxorpd %%ymm7, %%ymm7, %%ymm7 \n\t" + "vxorpd %%ymm8, %%ymm8, %%ymm8 \n\t" + "vxorpd %%ymm9, %%ymm9, %%ymm9 \n\t" + "vxorpd %%ymm10, %%ymm10, %%ymm10 \n\t" + "vxorpd %%ymm11, %%ymm11, %%ymm11 \n\t" + "vxorpd %%ymm12, %%ymm12, %%ymm12 \n\t" + "vxorpd %%ymm13, %%ymm13, %%ymm13 \n\t" + "vxorpd %%ymm14, %%ymm14, %%ymm14 \n\t" + "vxorpd %%ymm15, %%ymm15, %%ymm15 \n\t" + " \n\t" + " \n\t" + " \n\t" + "movq %0, %%rsi \n\t" // i = k_iter; + "testq %%rsi, %%rsi \n\t" // check i via logical AND. + "je .DCONSIDKLEFT \n\t" // if i == 0, jump to code that + " \n\t" // contains the k_left loop. + " \n\t" + " \n\t" + ".DLOOPKITER: \n\t" // MAIN LOOP + " \n\t" + " \n\t" + " \n\t" // iteration 0 + "prefetcht0 16 * 32(%%rax) \n\t" + " \n\t" + "vbroadcastsd 0 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 1 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm4 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm5 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm6 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm7 \n\t" + " \n\t" + "vbroadcastsd 2 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 3 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm8 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm9 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm11 \n\t" + " \n\t" + "vbroadcastsd 4 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 5 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm12 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm13 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm14 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm15 \n\t" + " \n\t" + "vmovaps -2 * 32(%%rax), %%ymm0 \n\t" + "vmovaps -1 * 32(%%rax), %%ymm1 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" // iteration 1 + "vbroadcastsd 6 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 7 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm4 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm5 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm6 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm7 \n\t" + " \n\t" + "vbroadcastsd 8 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 9 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm8 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm9 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm11 \n\t" + " \n\t" + "vbroadcastsd 10 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 11 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm12 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm13 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm14 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm15 \n\t" + " \n\t" + "vmovaps 0 * 32(%%rax), %%ymm0 \n\t" + "vmovaps 1 * 32(%%rax), %%ymm1 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" // iteration 2 + "prefetcht0 20 * 32(%%rax) \n\t" + " \n\t" + "vbroadcastsd 12 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 13 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm4 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm5 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm6 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm7 \n\t" + " \n\t" + "vbroadcastsd 14 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 15 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm8 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm9 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm11 \n\t" + " \n\t" + "vbroadcastsd 16 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 17 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm12 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm13 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm14 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm15 \n\t" + " \n\t" + "vmovaps 2 * 32(%%rax), %%ymm0 \n\t" + "vmovaps 3 * 32(%%rax), %%ymm1 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" // iteration 3 + "vbroadcastsd 18 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 19 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm4 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm5 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm6 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm7 \n\t" + " \n\t" + "vbroadcastsd 20 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 21 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm8 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm9 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm11 \n\t" + " \n\t" + "vbroadcastsd 22 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 23 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm12 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm13 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm14 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm15 \n\t" + " \n\t" + "addq $4 * 8 * 8, %%rax \n\t" // a += 4*8 (unroll x mr) + "addq $4 * 6 * 8, %%rbx \n\t" // b += 4*6 (unroll x nr) + " \n\t" + "vmovaps -4 * 32(%%rax), %%ymm0 \n\t" + "vmovaps -3 * 32(%%rax), %%ymm1 \n\t" + " \n\t" + " \n\t" + "decq %%rsi \n\t" // i -= 1; + "jne .DLOOPKITER \n\t" // iterate again if i != 0. + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + ".DCONSIDKLEFT: \n\t" + " \n\t" + "movq %1, %%rsi \n\t" // i = k_left; + "testq %%rsi, %%rsi \n\t" // check i via logical AND. + "je .DPOSTACCUM \n\t" // if i == 0, we're done; jump to end. + " \n\t" // else, we prepare to enter k_left loop. + " \n\t" + " \n\t" + ".DLOOPKLEFT: \n\t" // EDGE LOOP + " \n\t" + "prefetcht0 16 * 32(%%rax) \n\t" + " \n\t" + "vbroadcastsd 0 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 1 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm4 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm5 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm6 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm7 \n\t" + " \n\t" + "vbroadcastsd 2 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 3 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm8 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm9 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm11 \n\t" + " \n\t" + "vbroadcastsd 4 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 5 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm12 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm13 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm14 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm15 \n\t" + " \n\t" + "addq $1 * 8 * 8, %%rax \n\t" // a += 1*8 (unroll x mr) + "addq $1 * 6 * 8, %%rbx \n\t" // b += 1*6 (unroll x nr) + " \n\t" + "vmovaps -4 * 32(%%rax), %%ymm0 \n\t" + "vmovaps -3 * 32(%%rax), %%ymm1 \n\t" + " \n\t" + " \n\t" + "decq %%rsi \n\t" // i -= 1; + "jne .DLOOPKLEFT \n\t" // iterate again if i != 0. + " \n\t" + " \n\t" + " \n\t" + ".DPOSTACCUM: \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "movq %4, %%rax \n\t" // load address of alpha + "movq %5, %%rbx \n\t" // load address of beta + "vbroadcastsd (%%rax), %%ymm0 \n\t" // load alpha and duplicate + "vbroadcastsd (%%rbx), %%ymm3 \n\t" // load beta and duplicate + " \n\t" + "vmulpd %%ymm0, %%ymm4, %%ymm4 \n\t" // scale by alpha + "vmulpd %%ymm0, %%ymm5, %%ymm5 \n\t" + "vmulpd %%ymm0, %%ymm6, %%ymm6 \n\t" + "vmulpd %%ymm0, %%ymm7, %%ymm7 \n\t" + "vmulpd %%ymm0, %%ymm8, %%ymm8 \n\t" + "vmulpd %%ymm0, %%ymm9, %%ymm9 \n\t" + "vmulpd %%ymm0, %%ymm10, %%ymm10 \n\t" + "vmulpd %%ymm0, %%ymm11, %%ymm11 \n\t" + "vmulpd %%ymm0, %%ymm12, %%ymm12 \n\t" + "vmulpd %%ymm0, %%ymm13, %%ymm13 \n\t" + "vmulpd %%ymm0, %%ymm14, %%ymm14 \n\t" + "vmulpd %%ymm0, %%ymm15, %%ymm15 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "movq %7, %%rsi \n\t" // load rs_c + "leaq (,%%rsi,8), %%rsi \n\t" // rsi = rs_c * sizeof(double) + " \n\t" + "leaq (%%rcx,%%rsi,4), %%rdx \n\t" // load address of c + 4*rs_c; + " \n\t" + "leaq (%%rsi,%%rsi,2), %%r13 \n\t" // r13 = 3*rs_c; + //"leaq (%%rsi,%%rsi,4), %%r15 \n\t" // r15 = 5*rs_c; + //"leaq (%%r13,%%rsi,4), %%r10 \n\t" // r10 = 7*rs_c; + " \n\t" + " \n\t" + " \n\t" + " \n\t" // determine if + " \n\t" // c % 32 == 0, AND + " \n\t" // 8*cs_c % 32 == 0, AND + " \n\t" // rs_c == 1 + " \n\t" // ie: aligned, ldim aligned, and + " \n\t" // column-stored + " \n\t" + "cmpq $8, %%rsi \n\t" // set ZF if (8*rs_c) == 8. + "sete %%bl \n\t" // bl = ( ZF == 1 ? 1 : 0 ); + "testq $31, %%rcx \n\t" // set ZF if c & 32 is zero. + "setz %%bh \n\t" // bh = ( ZF == 0 ? 1 : 0 ); + "testq $31, %%rdi \n\t" // set ZF if (8*cs_c) & 32 is zero. + "setz %%al \n\t" // al = ( ZF == 0 ? 1 : 0 ); + " \n\t" // and(bl,bh) followed by + " \n\t" // and(bh,al) will reveal result + " \n\t" + " \n\t" // now avoid loading C if beta == 0 + " \n\t" + "vxorpd %%ymm0, %%ymm0, %%ymm0 \n\t" // set ymm0 to zero. + "vucomisd %%xmm0, %%xmm3 \n\t" // set ZF if beta == 0. + "je .DBETAZERO \n\t" // if ZF = 1, jump to beta == 0 case + " \n\t" + " \n\t" + " \n\t" // check if aligned/column-stored + "andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. + "andb %%bh, %%al \n\t" // set ZF if bh & al == 1. + "jne .DCOLSTORED \n\t" // jump to column storage case + " \n\t" + " \n\t" + " \n\t" + ".DGENSTORED: \n\t" + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm4, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm6, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm8, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm10, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm12, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm14, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + " \n\t" + " \n\t" + "movq %%rdx, %%rcx \n\t" // rcx = c + 4*rs_c + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm5, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm7, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm9, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm11, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm13, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm15, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + " \n\t" + " \n\t" + " \n\t" + "jmp .DDONE \n\t" // jump to end. + " \n\t" + " \n\t" + " \n\t" + ".DCOLSTORED: \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213pd %%ymm4, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213pd %%ymm5, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213pd %%ymm6, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213pd %%ymm7, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213pd %%ymm8, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213pd %%ymm9, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213pd %%ymm10, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213pd %%ymm11, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213pd %%ymm12, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213pd %%ymm13, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213pd %%ymm14, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + //"addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213pd %%ymm15, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + //"addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + " \n\t" + "jmp .DDONE \n\t" // jump to end. + " \n\t" + " \n\t" + " \n\t" + ".DBETAZERO: \n\t" + " \n\t" // check if aligned/column-stored + "andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. + "andb %%bh, %%al \n\t" // set ZF if bh & al == 1. + "jne .DCOLSTORBZ \n\t" // jump to column storage case + " \n\t" + " \n\t" + " \n\t" + ".DGENSTORBZ: \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm4, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm6, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm8, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm10, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm12, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm14, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + " \n\t" + " \n\t" + "movq %%rdx, %%rcx \n\t" // rcx = c + 4*rs_c + " \n\t" + " \n\t" + "vmovaps %%ymm5, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm7, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm9, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm11, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm13, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm15, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + " \n\t" + " \n\t" + " \n\t" + "jmp .DDONE \n\t" // jump to end. + " \n\t" + " \n\t" + " \n\t" + ".DCOLSTORBZ: \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm4, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm5, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + "vmovaps %%ymm6, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm7, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm8, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm9, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm10, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm11, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm12, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm13, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm14, (%%rcx) \n\t" + " \n\t" + "vmovaps %%ymm15, (%%rdx) \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + ".DDONE: \n\t" + " \n\t" + + : // output operands (none) + : // input operands + "m" (k_iter), // 0 + "m" (k_left), // 1 + "m" (a), // 2 + "m" (b), // 3 + "m" (alpha), // 4 + "m" (beta), // 5 + "m" (c), // 6 + "m" (rs_c), // 7 + "m" (cs_c)/*, // 8 + "m" (b_next), // 9 + "m" (a_next)*/ // 10 + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm6", "xmm7", + "xmm8", "xmm9", "xmm10", "xmm11", + "xmm12", "xmm13", "xmm14", "xmm15", + "memory" + ); +} + +#if 0 + +void bli_cgemm_asm_( + dim_t k, + scomplex* restrict alpha, + scomplex* restrict a, + scomplex* restrict b, + scomplex* restrict beta, + scomplex* restrict c, inc_t rs_c, inc_t cs_c, + auxinfo_t* data + ) +{ + //void* a_next = bli_auxinfo_next_a( data ); + //void* b_next = bli_auxinfo_next_b( data ); + + //dim_t k_iter = k / 4; + //dim_t k_left = k % 4; + +} + + + +void bli_zgemm_asm_( + dim_t k, + dcomplex* restrict alpha, + dcomplex* restrict a, + dcomplex* restrict b, + dcomplex* restrict beta, + dcomplex* restrict c, inc_t rs_c, inc_t cs_c, + auxinfo_t* data + ) +{ + //void* a_next = bli_auxinfo_next_a( data ); + //void* b_next = bli_auxinfo_next_b( data ); + + //dim_t k_iter = k / 4; + //dim_t k_left = k % 4; + +} + +#endif diff --git a/testsuite/src/test_gemm.c b/testsuite/src/test_gemm.c index 0d5732f524..43e619c579 100644 --- a/testsuite/src/test_gemm.c +++ b/testsuite/src/test_gemm.c @@ -174,12 +174,12 @@ void libblis_test_gemm_experiment( test_params_t* params, if ( bli_obj_is_real( c ) ) { bli_setsc( 1.2, 0.0, &alpha ); - bli_setsc( -1.0, 0.0, &beta ); + bli_setsc( 0.9, 0.0, &beta ); } else { bli_setsc( 1.2, 0.8, &alpha ); - bli_setsc( -1.0, 1.0, &beta ); + bli_setsc( 0.9, 1.0, &beta ); } // Randomize A, B, and C, and save C. From 47caa33485b91ea6f2a5e386e61210c90c5f489f Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Wed, 29 Jul 2015 13:31:09 -0500 Subject: [PATCH 14/38] Version file update (0.1.8) --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 11808190d4..699c6c6d4e 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.1.7 +0.1.8 From ecc3ebb749e0861c27deda52b5f87236ede4901b Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Wed, 29 Jul 2015 13:31:12 -0500 Subject: [PATCH 15/38] CHANGELOG update (0.1.8) --- CHANGELOG | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index e579f7f4a4..1c6b42d14f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,10 +1,58 @@ -commit 267253de8a7be546ce87626443ee38701c1d411f (HEAD -> master, tag: 0.1.7) +commit 47caa33485b91ea6f2a5e386e61210c90c5f489f (HEAD -> master, tag: 0.1.8) +Author: Field G. Van Zee +Date: Wed Jul 29 13:31:09 2015 -0500 + + Version file update (0.1.8) + +commit ef0fbbbdb6148b96938733fce72cb4ed7dad685e (origin/master) +Merge: fdfe14f d4b8913 +Author: Field G. Van Zee +Date: Thu Jul 9 13:54:54 2015 -0500 + + Merge branch 'master' of github.com:flame/blis + +commit fdfe14f1e17ba5a2f8dfa0bdb799c6b0e730211b +Author: Field G. Van Zee +Date: Thu Jul 9 13:52:39 2015 -0500 + + Added support for Intel Haswell/Broadwell. + + Details: + - Added sgemm and dgemm micro-kernels, which employ 256-bit AVX vectors + and FMA instructions. (Complex support is currently provided by default + induced method, 4m1a.) + - Added a 'haswell' configuration, which uses the aforementioned kernels. + - Inserted auto-detection support for haswell configuration in + build/auto-detect/cpuid_x86.c. + - Modified configure script to explicitly echo when automatic or manual + configuration is in progress. + - Changed beta scalar in test_gemm.c module of test suite to -1.0 to 0.9. + +commit d4b891369c1eb0879ade662ff896a5b9a7fca207 +Author: Field G. Van Zee +Date: Tue Jul 7 10:06:53 2015 -0500 + + Added 'carrizo' configuration. + + Details: + - Added a new configuration for AMD Excavator-based hardware also known + as Carrizo when referring to the entire APU. This configuration uses + the same micro-kernels as the piledriver, but with different + cache blocksizes. + +commit 0b7255a642d56723f02d7ca1f8f21809967b8515 +Author: Field G. Van Zee +Date: Fri Jun 19 12:01:50 2015 -0500 + + CHANGELOG update (0.1.7) + +commit 267253de8a7be546ce87626443ee38701c1d411f (tag: 0.1.7) Author: Field G. Van Zee Date: Fri Jun 19 12:01:49 2015 -0500 Version file update (0.1.7) -commit 7cd01b71b5e757a6774625b3c9f427f5e7664a76 (origin/master, origin/HEAD) +commit 7cd01b71b5e757a6774625b3c9f427f5e7664a76 Author: Field G. Van Zee Date: Fri Jun 19 11:31:53 2015 -0500 From 12ffd568b04feda57147c13b67717416a01c82f8 Mon Sep 17 00:00:00 2001 From: Zhang Xianyi Date: Sat, 22 Aug 2015 00:24:28 +0800 Subject: [PATCH 16/38] Add Travis CI. --- .travis.yml | 34 ++++++++++++++++++++++++++++++ build/check-test.sh | 50 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 .travis.yml create mode 100755 build/check-test.sh diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000..f325324650 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,34 @@ +language: c +sudo: false + +compiler: + - gcc + +env: + - RUN_TEST=1 BUILD_CONFIG="auto" + - RUN_TEST=1 BUILD_CONFIG="reference" + - RUN_TEST=1 BUILD_CONFIG="dunnington" + - RUN_TEST=0 BUILD_CONFIG="sandybridge" + - RUN_TEST=0 BUILD_CONFIG="haswell" + - RUN_TEST=0 BUILD_CONFIG="bulldozer" + - RUN_TEST=0 BUILD_CONFIG="piledriver" + - RUN_TEST=0 BUILD_CONFIG="carrizo" + +install: +- if [ "$CC" = "gcc" ]; then export CXX="g++-4.8" CC="gcc-4.8"; fi +addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - gcc-4.8 + - g++-4.8 + - clang + + + +script: + - ./configure $BUILD_CONFIG + - make + - if [ $RUN_TEST -eq 1 ]; then make test; fi + - if [ $RUN_TEST -eq 1 ]; then ./build/check-test.sh ./output.testsuite; fi \ No newline at end of file diff --git a/build/check-test.sh b/build/check-test.sh new file mode 100755 index 0000000000..6277ada452 --- /dev/null +++ b/build/check-test.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# +# BLIS +# An object-based framework for developing high-performance BLAS-like +# libraries. +# +# Copyright (C) 2015, The University of Texas at Austin +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# - Neither the name of The University of Texas at Austin nor the names +# of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# +# check-test.sh +# +# Zhang Xianyi +# + + +grep -q FAILURE $1 + +if [ $? -eq 0 ]; then + echo "Test Failure" + exit 1 +else + echo "Test Pass" + exit 0 +fi \ No newline at end of file From 4dd9dd3e1de626b51bfe85d9ee65f193d60e8d38 Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Fri, 21 Aug 2015 11:52:37 -0500 Subject: [PATCH 17/38] Fixed minor alignment ambiguity bug in bli_pool.c. Details: - Fixed a typecasting ambiguity in bli_pool_alloc_block() in which pointer arithmetic was performed on a void* as if it were a byte pointer (such as char*). Some compilers may have already been interpreting this situation as intended, despite the sloppiness. Thanks to Aleksei Rechinskii for reporting this issue. - Redefined pointer alignment macros to typecast to uintptr_t instead of siz_t. --- frame/base/bli_pool.c | 21 +++++++++++++++------ frame/include/bli_param_macro_defs.h | 6 +++--- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/frame/base/bli_pool.c b/frame/base/bli_pool.c index 9c7230faa7..305cd32345 100644 --- a/frame/base/bli_pool.c +++ b/frame/base/bli_pool.c @@ -282,13 +282,22 @@ void bli_pool_alloc_block( siz_t block_size, buf_sys = bli_malloc( block_size + align_size ); buf_align = buf_sys; - // Advance the pointer to achieve the necessary alignment, if it - // is not already aligned. - if ( bli_is_unaligned_to( ( uintptr_t )buf_sys, ( uintptr_t )align_size ) ) + // Advance the pointer to achieve the necessary alignment, if it is not + // already aligned. + if ( bli_is_unaligned_to( buf_sys, align_size ) ) { - // Notice that this works even if the alignment is not a power of two. - buf_align += ( ( uintptr_t )align_size - - ( ( uintptr_t )buf_sys % align_size ) ); + // C99's stdint.h guarantees that a void* can be safely cast to a + // uintptr_t and then back to a void*, hence the casting of buf_sys + // and align_size to uintptr_t. buf_align is initially cast to char* + // to allow pointer arithmetic in units of bytes, and then advanced + // to the next nearest alignment boundary, and finally cast back to + // void* before being stored. Notice that the arithmetic works even + // if the alignment value is not a power of two. + buf_align = ( void* )( ( char* )buf_align + + ( ( uintptr_t )align_size - + ( uintptr_t )buf_sys % + ( uintptr_t )align_size ) + ); } // Save the results in the pblk_t structure. diff --git a/frame/include/bli_param_macro_defs.h b/frame/include/bli_param_macro_defs.h index f1924d9209..40a054ffa9 100644 --- a/frame/include/bli_param_macro_defs.h +++ b/frame/include/bli_param_macro_defs.h @@ -42,15 +42,15 @@ #define bli_is_aligned_to( p, size ) \ \ - ( ( siz_t )(p) % ( siz_t )(size) == 0 ) + ( ( uintptr_t )(p) % ( uintptr_t )(size) == 0 ) #define bli_is_unaligned_to( p, size ) \ \ - ( ( siz_t )(p) % ( siz_t )(size) != 0 ) + ( ( uintptr_t )(p) % ( uintptr_t )(size) != 0 ) #define bli_offset_from_alignment( p, size ) \ \ - ( ( siz_t )(p) % ( siz_t )(size) ) + ( ( uintptr_t )(p) % ( uintptr_t )(size) ) // datatype From efa641e36b73abee34166a252e90e28a6281d92d Mon Sep 17 00:00:00 2001 From: Zhang Xianyi Date: Sat, 22 Aug 2015 03:15:50 +0800 Subject: [PATCH 18/38] Try to fix the compiling bug on travis. --- .travis.yml | 4 ++-- Makefile | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f325324650..bfe7412a58 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,6 @@ addons: script: - ./configure $BUILD_CONFIG - - make - - if [ $RUN_TEST -eq 1 ]; then make test; fi + - make CC=gcc-4.8 + - if [ $RUN_TEST -eq 1 ]; then make BLIS_ENABLE_TEST_OUTPUT=yes test; fi - if [ $RUN_TEST -eq 1 ]; then ./build/check-test.sh ./output.testsuite; fi \ No newline at end of file diff --git a/Makefile b/Makefile index 9ad0d75686..c9fe957b51 100644 --- a/Makefile +++ b/Makefile @@ -633,6 +633,11 @@ ifeq ($(BLIS_ENABLE_VERBOSE_MAKE_OUTPUT),yes) ./$(TESTSUITE_BIN) -g $(TESTSUITE_CONF_GEN_PATH) \ -o $(TESTSUITE_CONF_OPS_PATH) \ > $(TESTSUITE_OUT_FILE) + +else ifeq ($(BLIS_ENABLE_TEST_OUTPUT), yes) + ./$(TESTSUITE_BIN) -g $(TESTSUITE_CONF_GEN_PATH) \ + -o $(TESTSUITE_CONF_OPS_PATH) | \ + tee $(TESTSUITE_OUT_FILE) else @echo "Running $(TESTSUITE_BIN) with output redirected to '$(TESTSUITE_OUT_FILE)'" @./$(TESTSUITE_BIN) -g $(TESTSUITE_CONF_GEN_PATH) \ From e2e9d64a63485461192d9c2a6dd0183a8b71013c Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Thu, 24 Sep 2015 12:14:03 -0500 Subject: [PATCH 19/38] Load balance thread ranges for arbitrary diagonals. Details: - Expanded/updated interface for bli_get_range_weighted() and bli_get_range() so that the direction of movement is specified in the function name (e.g. bli_get_range_l2r(), bli_get_range_weighted_t2b()) and also so that the object being partitioned is passed instead of an uplo parameter. Updated invocations in level-3 blocked variants, as appropriate. - (Re)implemented bli_get_range_*() and bli_get_range_weighted_*() to carefully take into account the location of the diagonal when computing ranges so that the area of each subpartition (which, in all present level-3 operations, is proportional to the amount of computation engendered) is as equal as possible. - Added calls to a new class of routines to all non-gemm level-3 blocked variants: bli__prune_unref_mparts_[mnk]() where is herk, trmm, or trsm and [mnk] is chosen based on which dimension is being partitioned. These routines call a more basic routine, bli_prune_unref_mparts(), to prune unreferenced/unstored regions from matrices and simultaneously adjust other matrices which share the same dimension accordingly. - Simplified herk_blk_var2f, trmm_blk_var1f/b as a result of more the new pruning routines. - Fixed incorrect blocking factors passed into bli_get_range_*() in bli_trsm_blk_var[12][fb].c - Added a new test driver in test/thread_ranges that can exercise the new bli_get_range_*() and bli_get_range_weighted_*() under a range of conditions. - Reimplemented m and n fields of obj_t as elements in a "dim" array field so that dimensions could be queried via index constant (e.g. BLIS_M, BLIS_N). Adjusted/added query and modification macros accordingly. - Defined mdim_t type to enumerate BLIS_M and BLIS_N indexing values. - Added bli_round() macro, which calls C math library function round(), and bli_round_to_mult(), which rounds a value to the nearest multiple of some other value. - Added miscellaneous pruning- and mdim_t-related macros. - Renamed bli_obj_row_offset(), bli_obj_col_offset() macros to bli_obj_row_off(), bli_obj_col_off(). --- frame/3/gemm/bli_gemm_blk_var1f.c | 13 +- frame/3/gemm/bli_gemm_blk_var2f.c | 13 +- frame/3/gemm/ind/bli_gemm_blk_var4f.c | 13 +- frame/3/herk/bli_herk.h | 1 + frame/3/herk/bli_herk_blk_var1f.c | 16 +- frame/3/herk/bli_herk_blk_var2f.c | 52 +-- frame/3/herk/bli_herk_blk_var3f.c | 3 + frame/3/herk/bli_herk_prune.c | 64 +++ frame/3/herk/bli_herk_prune.h | 38 ++ frame/3/trmm/bli_trmm.h | 1 + frame/3/trmm/bli_trmm_blk_var1f.c | 31 +- frame/3/trmm/bli_trmm_blk_var2b.c | 15 +- frame/3/trmm/bli_trmm_blk_var2f.c | 15 +- frame/3/trmm/bli_trmm_blk_var3b.c | 3 + frame/3/trmm/bli_trmm_blk_var3f.c | 3 + frame/3/trmm/bli_trmm_prune.c | 71 ++++ frame/3/trmm/bli_trmm_prune.h | 38 ++ frame/3/trsm/bli_trsm.h | 1 + frame/3/trsm/bli_trsm_blk_var1b.c | 30 +- frame/3/trsm/bli_trsm_blk_var1f.c | 29 +- frame/3/trsm/bli_trsm_blk_var2b.c | 25 +- frame/3/trsm/bli_trsm_blk_var2f.c | 25 +- frame/3/trsm/bli_trsm_blk_var3b.c | 3 + frame/3/trsm/bli_trsm_blk_var3f.c | 3 + frame/3/trsm/bli_trsm_prune.c | 71 ++++ frame/3/trsm/bli_trsm_prune.h | 38 ++ frame/base/bli_obj.c | 4 +- frame/base/bli_part.c | 10 + frame/base/bli_prune.c | 135 ++++++ frame/base/bli_prune.h | 36 ++ frame/base/bli_threading.c | 579 ++++++++++++++++++++------ frame/base/bli_threading.h | 86 ++-- frame/include/bli_obj_macro_defs.h | 124 ++++-- frame/include/bli_param_macro_defs.h | 103 ++++- frame/include/bli_scalar_macro_defs.h | 16 + frame/include/bli_type_defs.h | 27 +- frame/include/blis.h | 1 + test/thread_ranges/Makefile | 203 +++++++++ test/thread_ranges/test_ranges.c | 313 ++++++++++++++ 39 files changed, 1875 insertions(+), 377 deletions(-) create mode 100644 frame/3/herk/bli_herk_prune.c create mode 100644 frame/3/herk/bli_herk_prune.h create mode 100644 frame/3/trmm/bli_trmm_prune.c create mode 100644 frame/3/trmm/bli_trmm_prune.h create mode 100644 frame/3/trsm/bli_trsm_prune.c create mode 100644 frame/3/trsm/bli_trsm_prune.h create mode 100644 frame/base/bli_prune.c create mode 100644 frame/base/bli_prune.h create mode 100644 test/thread_ranges/Makefile create mode 100644 test/thread_ranges/test_ranges.c diff --git a/frame/3/gemm/bli_gemm_blk_var1f.c b/frame/3/gemm/bli_gemm_blk_var1f.c index 535cc4710c..f3dfa284b2 100644 --- a/frame/3/gemm/bli_gemm_blk_var1f.c +++ b/frame/3/gemm/bli_gemm_blk_var1f.c @@ -51,7 +51,6 @@ void bli_gemm_blk_var1f( obj_t* a, dim_t i; dim_t b_alg; - dim_t m_trans; if( thread_am_ochief( thread ) ) { // Initialize object for packing B. @@ -80,21 +79,19 @@ void bli_gemm_blk_var1f( obj_t* a, cntl_sub_packm_b( cntl ), gemm_thread_sub_opackm( thread ) ); - // Query dimension in partitioning direction. - m_trans = bli_obj_length_after_trans( *a ); - dim_t start, end; - bli_get_range_t2b( thread, 0, m_trans, + dim_t my_start, my_end; + bli_get_range_t2b( thread, a, bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), - &start, &end ); + &my_start, &my_end ); // Partition along the m dimension. - for ( i = start; i < end; i += b_alg ) + for ( i = my_start; i < my_end; i += b_alg ) { // Determine the current algorithmic blocksize. // NOTE: Use of a (for execution datatype) is intentional! // This causes the right blocksize to be used if c and a are // complex and b is real. - b_alg = bli_determine_blocksize_f( i, end, a, + b_alg = bli_determine_blocksize_f( i, my_end, a, cntl_blocksize( cntl ) ); // Acquire partitions for A1 and C1. diff --git a/frame/3/gemm/bli_gemm_blk_var2f.c b/frame/3/gemm/bli_gemm_blk_var2f.c index f122f9a710..4b3d9da739 100644 --- a/frame/3/gemm/bli_gemm_blk_var2f.c +++ b/frame/3/gemm/bli_gemm_blk_var2f.c @@ -50,7 +50,6 @@ void bli_gemm_blk_var2f( obj_t* a, dim_t i; dim_t b_alg; - dim_t n_trans; if( thread_am_ochief( thread ) ) { @@ -79,21 +78,19 @@ void bli_gemm_blk_var2f( obj_t* a, cntl_sub_packm_a( cntl ), gemm_thread_sub_opackm( thread ) ); - // Query dimension in partitioning direction. - n_trans = bli_obj_width_after_trans( *b ); - dim_t start, end; - bli_get_range_l2r( thread, 0, n_trans, + dim_t my_start, my_end; + bli_get_range_l2r( thread, b, bli_blksz_get_mult_for_obj( b, cntl_blocksize( cntl ) ), - &start, &end ); + &my_start, &my_end ); // Partition along the n dimension. - for ( i = start; i < end; i += b_alg ) + for ( i = my_start; i < my_end; i += b_alg ) { // Determine the current algorithmic blocksize. // NOTE: Use of b (for execution datatype) is intentional! // This causes the right blocksize to be used if c and a are // complex and b is real. - b_alg = bli_determine_blocksize_f( i, end, b, + b_alg = bli_determine_blocksize_f( i, my_end, b, cntl_blocksize( cntl ) ); // Acquire partitions for B1 and C1. diff --git a/frame/3/gemm/ind/bli_gemm_blk_var4f.c b/frame/3/gemm/ind/bli_gemm_blk_var4f.c index d57c8c82fa..0102675be1 100644 --- a/frame/3/gemm/ind/bli_gemm_blk_var4f.c +++ b/frame/3/gemm/ind/bli_gemm_blk_var4f.c @@ -59,7 +59,6 @@ void bli_gemm_blk_var4f( obj_t* a, dim_t i; dim_t b_alg; - dim_t m_trans; if( thread_am_ochief( thread ) ) { // Initialize object for packing B. @@ -88,21 +87,19 @@ void bli_gemm_blk_var4f( obj_t* a, cntl_sub_packm_b( cntl ), gemm_thread_sub_opackm( thread ) ); - // Query dimension in partitioning direction. - m_trans = bli_obj_length_after_trans( *a ); - dim_t start, end; - bli_get_range_t2b( thread, 0, m_trans, + dim_t my_start, my_end; + bli_get_range_t2b( thread, a, bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), - &start, &end ); + &my_start, &my_end ); // Partition along the m dimension. - for ( i = start; i < end; i += b_alg ) + for ( i = my_start; i < my_end; i += b_alg ) { // Determine the current algorithmic blocksize. // NOTE: Use of a (for execution datatype) is intentional! // This causes the right blocksize to be used if c and a are // complex and b is real. - b_alg = bli_determine_blocksize_f( i, end, a, + b_alg = bli_determine_blocksize_f( i, my_end, a, cntl_blocksize( cntl ) ); // Acquire partitions for A1 and C1. diff --git a/frame/3/herk/bli_herk.h b/frame/3/herk/bli_herk.h index 2387a88243..1129d3ff65 100644 --- a/frame/3/herk/bli_herk.h +++ b/frame/3/herk/bli_herk.h @@ -35,6 +35,7 @@ #include "bli_herk_check.h" #include "bli_herk_front.h" #include "bli_herk_int.h" +#include "bli_herk_prune.h" #include "bli_herk_blk_var1f.h" diff --git a/frame/3/herk/bli_herk_blk_var1f.c b/frame/3/herk/bli_herk_blk_var1f.c index b982de34dc..a733d29b7e 100644 --- a/frame/3/herk/bli_herk_blk_var1f.c +++ b/frame/3/herk/bli_herk_blk_var1f.c @@ -50,7 +50,9 @@ void bli_herk_blk_var1f( obj_t* a, dim_t i; dim_t b_alg; - dim_t m_trans; + + // Prune any zero region that exists along the partitioning dimension. + bli_herk_prune_unref_mparts_m( a, ah, c ); if( thread_am_ochief( thread ) ) { // Initialize object for packing A'. @@ -79,18 +81,16 @@ void bli_herk_blk_var1f( obj_t* a, cntl_sub_packm_b( cntl ), herk_thread_sub_opackm( thread ) ); - // Query dimension in partitioning direction. - m_trans = bli_obj_length_after_trans( *c ); - dim_t start, end; - bli_get_range_weighted_t2b( thread, 0, m_trans, + dim_t my_start, my_end; + bli_get_range_weighted_t2b( thread, c, bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), - bli_obj_root_uplo( *c ), &start, &end ); + &my_start, &my_end ); // Partition along the m dimension. - for ( i = start; i < end; i += b_alg ) + for ( i = my_start; i < my_end; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_f( i, end, a, + b_alg = bli_determine_blocksize_f( i, my_end, a, cntl_blocksize( cntl ) ); // Acquire partitions for A1 and C1. diff --git a/frame/3/herk/bli_herk_blk_var2f.c b/frame/3/herk/bli_herk_blk_var2f.c index b89842c408..b4a0ea924e 100644 --- a/frame/3/herk/bli_herk_blk_var2f.c +++ b/frame/3/herk/bli_herk_blk_var2f.c @@ -41,23 +41,18 @@ void bli_herk_blk_var2f( obj_t* a, herk_thrinfo_t* thread ) { obj_t a_pack_s; - obj_t ah1_pack_s, c1S_pack_s; + obj_t ah1_pack_s, c1_pack_s; - obj_t ah1, c1, c1S; - obj_t aS_pack; + obj_t ah1, c1; obj_t* a_pack; obj_t* ah1_pack; - obj_t* c1S_pack; + obj_t* c1_pack; dim_t i; dim_t b_alg; - dim_t n_trans; - subpart_t stored_part; - // The upper and lower variants are identical, except for which - // merged subpartition is acquired in the loop body. - if ( bli_obj_is_lower( *c ) ) stored_part = BLIS_SUBPART1B; - else stored_part = BLIS_SUBPART1T; + // Prune any zero region that exists along the partitioning dimension. + bli_herk_prune_unref_mparts_n( a, ah, c ); if( thread_am_ochief( thread ) ) { // Initialize object for packing A @@ -75,30 +70,26 @@ void bli_herk_blk_var2f( obj_t* a, // Initialize pack objects for C and A' that are passed into packm_init(). if( thread_am_ichief( thread ) ) { bli_obj_init_pack( &ah1_pack_s ); - bli_obj_init_pack( &c1S_pack_s ); + bli_obj_init_pack( &c1_pack_s ); } ah1_pack = thread_ibroadcast( thread, &ah1_pack_s ); - c1S_pack = thread_ibroadcast( thread, &c1S_pack_s ); + c1_pack = thread_ibroadcast( thread, &c1_pack_s ); // Pack A (if instructed). bli_packm_int( a, a_pack, cntl_sub_packm_a( cntl ), herk_thread_sub_opackm( thread ) ); - // Query dimension in partitioning direction. - n_trans = bli_obj_width_after_trans( *c ); - dim_t start, end; - - // Needs to be replaced with a weighted range because triangle - bli_get_range_weighted_l2r( thread, 0, n_trans, + dim_t my_start, my_end; + bli_get_range_weighted_l2r( thread, c, bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), - bli_obj_root_uplo( *c ), &start, &end ); + &my_start, &my_end ); // Partition along the n dimension. - for ( i = start; i < end; i += b_alg ) + for ( i = my_start; i < my_end; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_f( i, end, a, + b_alg = bli_determine_blocksize_f( i, my_end, a, cntl_blocksize( cntl ) ); // Acquire partitions for A1' and C1. @@ -107,18 +98,11 @@ void bli_herk_blk_var2f( obj_t* a, bli_acquire_mpart_l2r( BLIS_SUBPART1, i, b_alg, c, &c1 ); - // Partition off the stored region of C1 and the corresponding region - // of A_pack. - bli_acquire_mpart_t2b( stored_part, - i, b_alg, &c1, &c1S ); - bli_acquire_mpart_t2b( stored_part, - i, b_alg, a_pack, &aS_pack ); - // Initialize objects for packing A1' and C1. if( thread_am_ichief( thread ) ) { bli_packm_init( &ah1, ah1_pack, cntl_sub_packm_b( cntl ) ); - bli_packm_init( &c1S, c1S_pack, + bli_packm_init( &c1, c1_pack, cntl_sub_packm_c( cntl ) ); } thread_ibarrier( thread ) ; @@ -129,23 +113,23 @@ void bli_herk_blk_var2f( obj_t* a, herk_thread_sub_ipackm( thread ) ); // Pack C1 (if instructed). - bli_packm_int( &c1S, c1S_pack, + bli_packm_int( &c1, c1_pack, cntl_sub_packm_c( cntl ), herk_thread_sub_ipackm( thread ) ) ; // Perform herk subproblem. bli_herk_int( &BLIS_ONE, - &aS_pack, + a_pack, ah1_pack, &BLIS_ONE, - c1S_pack, + c1_pack, cntl_sub_gemm( cntl ), herk_thread_sub_herk( thread ) ); thread_ibarrier( thread ); // Unpack C1 (if C1 was packed). - bli_unpackm_int( c1S_pack, &c1S, + bli_unpackm_int( c1_pack, &c1, cntl_sub_unpackm_c( cntl ), herk_thread_sub_ipackm( thread ) ); } @@ -157,7 +141,7 @@ void bli_herk_blk_var2f( obj_t* a, bli_packm_release( a_pack, cntl_sub_packm_a( cntl ) ); if( thread_am_ichief( thread ) ) { bli_packm_release( ah1_pack, cntl_sub_packm_b( cntl ) ); - bli_packm_release( c1S_pack, cntl_sub_packm_c( cntl ) ); + bli_packm_release( c1_pack, cntl_sub_packm_c( cntl ) ); } } diff --git a/frame/3/herk/bli_herk_blk_var3f.c b/frame/3/herk/bli_herk_blk_var3f.c index df5ec73be1..fd20c14cda 100644 --- a/frame/3/herk/bli_herk_blk_var3f.c +++ b/frame/3/herk/bli_herk_blk_var3f.c @@ -52,6 +52,9 @@ void bli_herk_blk_var3f( obj_t* a, dim_t b_alg; dim_t k_trans; + // Prune any zero region that exists along the partitioning dimension. + bli_herk_prune_unref_mparts_k( a, ah, c ); + if( thread_am_ochief( thread ) ) { // Initialize object for packing C. bli_obj_init_pack( &c_pack_s ); diff --git a/frame/3/herk/bli_herk_prune.c b/frame/3/herk/bli_herk_prune.c new file mode 100644 index 0000000000..a02c50a34b --- /dev/null +++ b/frame/3/herk/bli_herk_prune.c @@ -0,0 +1,64 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + +void bli_herk_prune_unref_mparts_m( obj_t* a, + obj_t* ah, + obj_t* c ) +{ + // Prune any unreferenced part from the subpartition of C (that would + // be encountered from partitioning in the m dimension) and adjust the + // subpartition of A accordingly. + bli_prune_unref_mparts( c, BLIS_M, a, BLIS_M ); +} + +void bli_herk_prune_unref_mparts_n( obj_t* a, + obj_t* ah, + obj_t* c ) +{ + // Prune any unreferenced part from the subpartition of C (that would + // be encountered from partitioning in the n dimension) and adjust the + // subpartition of Ah accordingly. + bli_prune_unref_mparts( c, BLIS_N, ah, BLIS_N ); +} + +void bli_herk_prune_unref_mparts_k( obj_t* a, + obj_t* ah, + obj_t* c ) +{ + // As long as A and Ah are general in structure, no pruning should be + // for the k dimension. +} + diff --git a/frame/3/herk/bli_herk_prune.h b/frame/3/herk/bli_herk_prune.h new file mode 100644 index 0000000000..4754d15846 --- /dev/null +++ b/frame/3/herk/bli_herk_prune.h @@ -0,0 +1,38 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +void bli_herk_prune_unref_mparts_m( obj_t* a, obj_t* ah, obj_t* c ); +void bli_herk_prune_unref_mparts_n( obj_t* a, obj_t* ah, obj_t* c ); +void bli_herk_prune_unref_mparts_k( obj_t* a, obj_t* ah, obj_t* c ); + diff --git a/frame/3/trmm/bli_trmm.h b/frame/3/trmm/bli_trmm.h index e621dd201e..09f407a9f5 100644 --- a/frame/3/trmm/bli_trmm.h +++ b/frame/3/trmm/bli_trmm.h @@ -36,6 +36,7 @@ #include "bli_trmm_check.h" #include "bli_trmm_front.h" #include "bli_trmm_int.h" +#include "bli_trmm_prune.h" #include "bli_trmm_blk_var1f.h" diff --git a/frame/3/trmm/bli_trmm_blk_var1f.c b/frame/3/trmm/bli_trmm_blk_var1f.c index 8095e0d523..d0127cbe82 100644 --- a/frame/3/trmm/bli_trmm_blk_var1f.c +++ b/frame/3/trmm/bli_trmm_blk_var1f.c @@ -50,8 +50,9 @@ void bli_trmm_blk_var1f( obj_t* a, dim_t i; dim_t b_alg; - dim_t m_trans; - dim_t offA; + + // Prune any zero region that exists along the partitioning dimension. + bli_trmm_prune_unref_mparts_m( a, b, c ); if( thread_am_ochief( thread ) ) { // Initialize object for packing B. @@ -81,28 +82,28 @@ void bli_trmm_blk_var1f( obj_t* a, trmm_thread_sub_opackm( thread ) ); // Set the default length of and offset to the non-zero part of A. - m_trans = bli_obj_length_after_trans( *a ); - offA = 0; + //m_trans = bli_obj_length_after_trans( *a ); + //offA = 0; // If A is lower triangular, we have to adjust where the non-zero part of // A begins. If A is upper triangular, we have to adjust the length of // the non-zero part. If A is general/dense, then we keep the defaults. - if ( bli_obj_is_lower( *a ) ) - offA = bli_abs( bli_obj_diag_offset_after_trans( *a ) ); - else if ( bli_obj_is_upper( *a ) ) - m_trans = bli_abs( bli_obj_diag_offset_after_trans( *a ) ) + - bli_obj_width_after_trans( *a ); - - dim_t start, end; - bli_get_range_weighted_t2b( thread, offA, m_trans, + //if ( bli_obj_is_lower( *a ) ) + // offA = bli_abs( bli_obj_diag_offset_after_trans( *a ) ); + //else if ( bli_obj_is_upper( *a ) ) + // m_trans = bli_abs( bli_obj_diag_offset_after_trans( *a ) ) + + // bli_obj_width_after_trans( *a ); + + dim_t my_start, my_end; + bli_get_range_weighted_t2b( thread, a, bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), - bli_obj_root_uplo( *a ), &start, &end ); + &my_start, &my_end ); // Partition along the m dimension. - for ( i = start; i < end; i += b_alg ) + for ( i = my_start; i < my_end; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_f( i, end, a, + b_alg = bli_determine_blocksize_f( i, my_end, a, cntl_blocksize( cntl ) ); // Acquire partitions for A1 and C1. diff --git a/frame/3/trmm/bli_trmm_blk_var2b.c b/frame/3/trmm/bli_trmm_blk_var2b.c index 33b79434bf..fe3752ffad 100644 --- a/frame/3/trmm/bli_trmm_blk_var2b.c +++ b/frame/3/trmm/bli_trmm_blk_var2b.c @@ -50,8 +50,9 @@ void bli_trmm_blk_var2b( obj_t* a, dim_t i; dim_t b_alg; - dim_t n_trans; + // Prune any zero region that exists along the partitioning dimension. + bli_trmm_prune_unref_mparts_n( a, b, c ); if( thread_am_ochief( thread ) ) { // Initialize object for packing A @@ -79,18 +80,16 @@ void bli_trmm_blk_var2b( obj_t* a, cntl_sub_packm_a( cntl ), trmm_thread_sub_opackm( thread ) ); - // Query dimension in partitioning direction. - n_trans = bli_obj_width_after_trans( *b ); - dim_t start, end; - bli_get_range_weighted_r2l( thread, 0, n_trans, + dim_t my_start, my_end; + bli_get_range_weighted_r2l( thread, b, bli_blksz_get_mult_for_obj( b, cntl_blocksize( cntl ) ), - bli_obj_root_uplo( *b ), &start, &end ); + &my_start, &my_end ); // Partition along the n dimension. - for ( i = start; i < end; i += b_alg ) + for ( i = my_start; i < my_end; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_b( i, end, b, + b_alg = bli_determine_blocksize_b( i, my_end, b, cntl_blocksize( cntl ) ); // Acquire partitions for B1 and C1. diff --git a/frame/3/trmm/bli_trmm_blk_var2f.c b/frame/3/trmm/bli_trmm_blk_var2f.c index 92f1ec0d2b..66ecb38400 100644 --- a/frame/3/trmm/bli_trmm_blk_var2f.c +++ b/frame/3/trmm/bli_trmm_blk_var2f.c @@ -50,8 +50,9 @@ void bli_trmm_blk_var2f( obj_t* a, dim_t i; dim_t b_alg; - dim_t n_trans; + // Prune any zero region that exists along the partitioning dimension. + bli_trmm_prune_unref_mparts_n( a, b, c ); if( thread_am_ochief( thread ) ) { // Initialize object for packing A @@ -79,18 +80,16 @@ void bli_trmm_blk_var2f( obj_t* a, cntl_sub_packm_a( cntl ), trmm_thread_sub_opackm( thread ) ); - // Query dimension in partitioning direction. - n_trans = bli_obj_width_after_trans( *b ); - dim_t start, end; - bli_get_range_weighted_l2r( thread, 0, n_trans, + dim_t my_start, my_end; + bli_get_range_weighted_l2r( thread, b, bli_blksz_get_mult_for_obj( b, cntl_blocksize( cntl ) ), - bli_obj_root_uplo( *b ), &start, &end ); + &my_start, &my_end ); // Partition along the n dimension. - for ( i = start; i < end; i += b_alg ) + for ( i = my_start; i < my_end; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_f( i, end, b, + b_alg = bli_determine_blocksize_f( i, my_end, b, cntl_blocksize( cntl ) ); // Acquire partitions for B1 and C1. diff --git a/frame/3/trmm/bli_trmm_blk_var3b.c b/frame/3/trmm/bli_trmm_blk_var3b.c index a9428edb62..e81073b93a 100644 --- a/frame/3/trmm/bli_trmm_blk_var3b.c +++ b/frame/3/trmm/bli_trmm_blk_var3b.c @@ -52,6 +52,9 @@ void bli_trmm_blk_var3b( obj_t* a, dim_t b_alg; dim_t k_trans; + // Prune any zero region that exists along the partitioning dimension. + bli_trmm_prune_unref_mparts_k( a, b, c ); + if( thread_am_ochief( thread ) ){ // Initialize object for packing C bli_obj_init_pack( &c_pack_s ); diff --git a/frame/3/trmm/bli_trmm_blk_var3f.c b/frame/3/trmm/bli_trmm_blk_var3f.c index cca0318f0a..2b1dfa2861 100644 --- a/frame/3/trmm/bli_trmm_blk_var3f.c +++ b/frame/3/trmm/bli_trmm_blk_var3f.c @@ -52,6 +52,9 @@ void bli_trmm_blk_var3f( obj_t* a, dim_t b_alg; dim_t k_trans; + // Prune any zero region that exists along the partitioning dimension. + bli_trmm_prune_unref_mparts_k( a, b, c ); + if( thread_am_ochief( thread ) ){ // Initialize object for packing C bli_obj_init_pack( &c_pack_s ); diff --git a/frame/3/trmm/bli_trmm_prune.c b/frame/3/trmm/bli_trmm_prune.c new file mode 100644 index 0000000000..04c0f621df --- /dev/null +++ b/frame/3/trmm/bli_trmm_prune.c @@ -0,0 +1,71 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + +void bli_trmm_prune_unref_mparts_m( obj_t* a, + obj_t* b, + obj_t* c ) +{ + // Prune any unreferenced part from the subpartition of A (that would + // be encountered from partitioning in the m dimension) and adjust the + // subpartition of C accordingly. + bli_prune_unref_mparts( a, BLIS_M, c, BLIS_M ); +} + +void bli_trmm_prune_unref_mparts_n( obj_t* a, + obj_t* b, + obj_t* c ) +{ + // Prune any unreferenced part from the subpartition of B (that would + // be encountered from partitioning in the n dimension) and adjust the + // subpartition of C accordingly. + bli_prune_unref_mparts( b, BLIS_N, c, BLIS_N ); +} + +void bli_trmm_prune_unref_mparts_k( obj_t* a, + obj_t* b, + obj_t* c ) +{ + // Prune any unreferenced part from the subpartition of A (that would + // be encountered from partitioning in the k dimension) and adjust the + // subpartition of B accordingly. + bli_prune_unref_mparts( a, BLIS_N, b, BLIS_M ); + + // Prune any unreferenced part from the subpartition of B (that would + // be encountered from partitioning in the k dimension) and adjust the + // subpartition of A accordingly. + bli_prune_unref_mparts( b, BLIS_M, a, BLIS_N ); +} + diff --git a/frame/3/trmm/bli_trmm_prune.h b/frame/3/trmm/bli_trmm_prune.h new file mode 100644 index 0000000000..1d065a58ea --- /dev/null +++ b/frame/3/trmm/bli_trmm_prune.h @@ -0,0 +1,38 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +void bli_trmm_prune_unref_mparts_m( obj_t* a, obj_t* b, obj_t* c ); +void bli_trmm_prune_unref_mparts_n( obj_t* a, obj_t* b, obj_t* c ); +void bli_trmm_prune_unref_mparts_k( obj_t* a, obj_t* b, obj_t* c ); + diff --git a/frame/3/trsm/bli_trsm.h b/frame/3/trsm/bli_trsm.h index bc60547528..6245d7286d 100644 --- a/frame/3/trsm/bli_trsm.h +++ b/frame/3/trsm/bli_trsm.h @@ -37,6 +37,7 @@ #include "bli_trsm_check.h" #include "bli_trsm_front.h" #include "bli_trsm_int.h" +#include "bli_trsm_prune.h" #include "bli_gemmtrsm_ukernel.h" #include "bli_trsm_ukernel.h" diff --git a/frame/3/trsm/bli_trsm_blk_var1b.c b/frame/3/trsm/bli_trsm_blk_var1b.c index a79a3033a0..29aad41df1 100644 --- a/frame/3/trsm/bli_trsm_blk_var1b.c +++ b/frame/3/trsm/bli_trsm_blk_var1b.c @@ -49,8 +49,9 @@ void bli_trsm_blk_var1b( obj_t* a, dim_t i; dim_t b_alg; - dim_t m_trans; - dim_t offA; + + // Prune any zero region that exists along the partitioning dimension. + bli_trsm_prune_unref_mparts_m( a, b, c ); // Initialize object for packing B. if( thread_am_ochief( thread ) ) { @@ -71,28 +72,19 @@ void bli_trsm_blk_var1b( obj_t* a, cntl_sub_packm_b( cntl ), trsm_thread_sub_opackm( thread ) ); - // Set the default length of and offset to the non-zero part of A. - m_trans = bli_obj_length_after_trans( *a ); - offA = 0; - - // If A is upper triangular, we have to adjust where the non-zero part of - // A begins. - if ( bli_obj_is_upper( *a ) ) - offA = m_trans - bli_abs( bli_obj_diag_offset_after_trans( *a ) ) - - bli_obj_width_after_trans( *a ); - - dim_t start, end; + dim_t my_start, my_end; num_t dt = bli_obj_execution_datatype( *a ); - bli_get_range_b2t( thread, offA, m_trans, - //bli_lcm( bli_info_get_default_nr( BLIS_TRSM, dt ), bli_info_get_default_mr( BLIS_TRSM, dt ) ), - bli_info_get_default_mc( BLIS_TRSM, dt ), - &start, &end ); + dim_t bf = ( bli_obj_root_is_triangular( *a ) ? + bli_info_get_default_mr( BLIS_TRSM, dt ) : + bli_info_get_default_nr( BLIS_TRSM, dt ) ); + bli_get_range_b2t( thread, a, bf, + &my_start, &my_end ); // Partition along the remaining portion of the m dimension. - for ( i = start; i < end; i += b_alg ) + for ( i = my_start; i < my_end; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_b( i, end, a, + b_alg = bli_determine_blocksize_b( i, my_end, a, cntl_blocksize( cntl ) ); // Acquire partitions for A1 and C1. diff --git a/frame/3/trsm/bli_trsm_blk_var1f.c b/frame/3/trsm/bli_trsm_blk_var1f.c index b246263329..b4a90f4634 100644 --- a/frame/3/trsm/bli_trsm_blk_var1f.c +++ b/frame/3/trsm/bli_trsm_blk_var1f.c @@ -49,8 +49,9 @@ void bli_trsm_blk_var1f( obj_t* a, dim_t i; dim_t b_alg; - dim_t m_trans; - dim_t offA; + + // Prune any zero region that exists along the partitioning dimension. + bli_trsm_prune_unref_mparts_m( a, b, c ); // Initialize object for packing B. if( thread_am_ochief( thread ) ) { @@ -71,27 +72,19 @@ void bli_trsm_blk_var1f( obj_t* a, cntl_sub_packm_b( cntl ), trsm_thread_sub_opackm( thread ) ); - // Set the default length of and offset to the non-zero part of A. - m_trans = bli_obj_length_after_trans( *a ); - offA = 0; - - // If A is lower triangular, we have to adjust where the non-zero part of - // A begins. - if ( bli_obj_is_lower( *a ) ) - offA = bli_abs( bli_obj_diag_offset_after_trans( *a ) ); - - dim_t start, end; + dim_t my_start, my_end; num_t dt = bli_obj_execution_datatype( *a ); - bli_get_range_t2b( thread, offA, m_trans, - //bli_lcm( bli_info_get_default_nr( BLIS_TRSM, dt ), bli_info_get_default_mr( BLIS_TRSM, dt ) ), - bli_info_get_default_mc( BLIS_TRSM, dt ), - &start, &end ); + dim_t bf = ( bli_obj_root_is_triangular( *a ) ? + bli_info_get_default_mr( BLIS_TRSM, dt ) : + bli_info_get_default_nr( BLIS_TRSM, dt ) ); + bli_get_range_t2b( thread, a, bf, + &my_start, &my_end ); // Partition along the remaining portion of the m dimension. - for ( i = start; i < end; i += b_alg ) + for ( i = my_start; i < my_end; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_f( i, end, a, + b_alg = bli_determine_blocksize_f( i, my_end, a, cntl_blocksize( cntl ) ); // Acquire partitions for A1 and C1. diff --git a/frame/3/trsm/bli_trsm_blk_var2b.c b/frame/3/trsm/bli_trsm_blk_var2b.c index b53c5884f0..6676528271 100644 --- a/frame/3/trsm/bli_trsm_blk_var2b.c +++ b/frame/3/trsm/bli_trsm_blk_var2b.c @@ -50,7 +50,9 @@ void bli_trsm_blk_var2b( obj_t* a, dim_t i; dim_t b_alg; - dim_t n_trans; + + // Prune any zero region that exists along the partitioning dimension. + bli_trsm_prune_unref_mparts_n( a, b, c ); // Initialize pack objects for A that are passed into packm_init(). if( thread_am_ochief( thread ) ) { @@ -78,24 +80,21 @@ void bli_trsm_blk_var2b( obj_t* a, // Pack A (if instructed). bli_packm_int( a, a_pack, cntl_sub_packm_a( cntl ), - trmm_thread_sub_opackm( thread ) ); + trsm_thread_sub_opackm( thread ) ); - // Query dimension in partitioning direction. - n_trans = bli_obj_width_after_trans( *b ); - dim_t start, end; + dim_t my_start, my_end; num_t dt = bli_obj_execution_datatype( *a ); - bli_get_range_r2l( thread, 0, n_trans, - //bli_lcm( bli_info_get_default_nr( BLIS_TRSM, dt ), - // bli_info_get_default_mr( BLIS_TRSM, dt ) ), - bli_lcm( bli_blksz_get_nr( dt, cntl_blocksize( cntl ) ), - bli_blksz_get_mr( dt, cntl_blocksize( cntl ) ) ), - &start, &end ); + dim_t bf = ( bli_obj_root_is_triangular( *b ) ? + bli_info_get_default_mr( BLIS_TRSM, dt ) : + bli_info_get_default_nr( BLIS_TRSM, dt ) ); + bli_get_range_r2l( thread, b, bf, + &my_start, &my_end ); // Partition along the n dimension. - for ( i = start; i < end; i += b_alg ) + for ( i = my_start; i < my_end; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_b( i, end, b, + b_alg = bli_determine_blocksize_b( i, my_end, b, cntl_blocksize( cntl ) ); // Acquire partitions for B1 and C1. diff --git a/frame/3/trsm/bli_trsm_blk_var2f.c b/frame/3/trsm/bli_trsm_blk_var2f.c index a770ec59db..955d11349d 100644 --- a/frame/3/trsm/bli_trsm_blk_var2f.c +++ b/frame/3/trsm/bli_trsm_blk_var2f.c @@ -50,7 +50,9 @@ void bli_trsm_blk_var2f( obj_t* a, dim_t i; dim_t b_alg; - dim_t n_trans; + + // Prune any zero region that exists along the partitioning dimension. + bli_trsm_prune_unref_mparts_n( a, b, c ); // Initialize pack objects for A that are passed into packm_init(). if( thread_am_ochief( thread ) ) { @@ -78,24 +80,21 @@ void bli_trsm_blk_var2f( obj_t* a, // Pack A (if instructed). bli_packm_int( a, a_pack, cntl_sub_packm_a( cntl ), - trmm_thread_sub_opackm( thread ) ); + trsm_thread_sub_opackm( thread ) ); - // Query dimension in partitioning direction. - n_trans = bli_obj_width_after_trans( *b ); - dim_t start, end; + dim_t my_start, my_end; num_t dt = bli_obj_execution_datatype( *a ); - bli_get_range_l2r( thread, 0, n_trans, - //bli_lcm( bli_info_get_default_nr( BLIS_TRSM, dt ), - // bli_info_get_default_mr( BLIS_TRSM, dt ) ), - bli_lcm( bli_blksz_get_nr( dt, cntl_blocksize( cntl ) ), - bli_blksz_get_mr( dt, cntl_blocksize( cntl ) ) ), - &start, &end ); + dim_t bf = ( bli_obj_root_is_triangular( *b ) ? + bli_info_get_default_mr( BLIS_TRSM, dt ) : + bli_info_get_default_nr( BLIS_TRSM, dt ) ); + bli_get_range_l2r( thread, b, bf, + &my_start, &my_end ); // Partition along the n dimension. - for ( i = start; i < end; i += b_alg ) + for ( i = my_start; i < my_end; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_f( i, end, b, + b_alg = bli_determine_blocksize_f( i, my_end, b, cntl_blocksize( cntl ) ); // Acquire partitions for B1 and C1. diff --git a/frame/3/trsm/bli_trsm_blk_var3b.c b/frame/3/trsm/bli_trsm_blk_var3b.c index 63b89a4846..411178487d 100644 --- a/frame/3/trsm/bli_trsm_blk_var3b.c +++ b/frame/3/trsm/bli_trsm_blk_var3b.c @@ -52,6 +52,9 @@ void bli_trsm_blk_var3b( obj_t* a, dim_t b_alg; dim_t k_trans; + // Prune any zero region that exists along the partitioning dimension. + bli_trsm_prune_unref_mparts_k( a, b, c ); + // Initialize pack objects for C that are passed into packm_init(). if( thread_am_ochief( thread ) ) { bli_obj_init_pack( &c_pack_s ); diff --git a/frame/3/trsm/bli_trsm_blk_var3f.c b/frame/3/trsm/bli_trsm_blk_var3f.c index afa9c52593..ce84201a2f 100644 --- a/frame/3/trsm/bli_trsm_blk_var3f.c +++ b/frame/3/trsm/bli_trsm_blk_var3f.c @@ -52,6 +52,9 @@ void bli_trsm_blk_var3f( obj_t* a, dim_t b_alg; dim_t k_trans; + // Prune any zero region that exists along the partitioning dimension. + bli_trsm_prune_unref_mparts_k( a, b, c ); + // Initialize pack objects for C that are passed into packm_init(). if( thread_am_ochief( thread ) ) { bli_obj_init_pack( &c_pack_s ); diff --git a/frame/3/trsm/bli_trsm_prune.c b/frame/3/trsm/bli_trsm_prune.c new file mode 100644 index 0000000000..768ea118d5 --- /dev/null +++ b/frame/3/trsm/bli_trsm_prune.c @@ -0,0 +1,71 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + +void bli_trsm_prune_unref_mparts_m( obj_t* a, + obj_t* b, + obj_t* c ) +{ + // Prune any unreferenced part from the subpartition of A (that would + // be encountered from partitioning in the m dimension) and adjust the + // subpartition of C accordingly. + bli_prune_unref_mparts( a, BLIS_M, c, BLIS_M ); +} + +void bli_trsm_prune_unref_mparts_n( obj_t* a, + obj_t* b, + obj_t* c ) +{ + // Prune any unreferenced part from the subpartition of B (that would + // be encountered from partitioning in the n dimension) and adjust the + // subpartition of C accordingly. + bli_prune_unref_mparts( b, BLIS_N, c, BLIS_N ); +} + +void bli_trsm_prune_unref_mparts_k( obj_t* a, + obj_t* b, + obj_t* c ) +{ + // Prune any unreferenced part from the subpartition of A (that would + // be encountered from partitioning in the k dimension) and adjust the + // subpartition of B accordingly. + bli_prune_unref_mparts( a, BLIS_N, b, BLIS_M ); + + // Prune any unreferenced part from the subpartition of B (that would + // be encountered from partitioning in the k dimension) and adjust the + // subpartition of A accordingly. + bli_prune_unref_mparts( b, BLIS_M, a, BLIS_N ); +} + diff --git a/frame/3/trsm/bli_trsm_prune.h b/frame/3/trsm/bli_trsm_prune.h new file mode 100644 index 0000000000..724db38aa9 --- /dev/null +++ b/frame/3/trsm/bli_trsm_prune.h @@ -0,0 +1,38 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +void bli_trsm_prune_unref_mparts_m( obj_t* a, obj_t* b, obj_t* c ); +void bli_trsm_prune_unref_mparts_n( obj_t* a, obj_t* b, obj_t* c ); +void bli_trsm_prune_unref_mparts_k( obj_t* a, obj_t* b, obj_t* c ); + diff --git a/frame/base/bli_obj.c b/frame/base/bli_obj.c index b6277dcb3e..75f2cd7e04 100644 --- a/frame/base/bli_obj.c +++ b/frame/base/bli_obj.c @@ -509,8 +509,8 @@ void bli_obj_print( char* label, obj_t* obj ) ( unsigned long int )bli_obj_width( *obj ) ); fprintf( file, "\n" ); - fprintf( file, " offm, offn %lu, %lu\n", ( unsigned long int )bli_obj_row_offset( *obj ), - ( unsigned long int )bli_obj_col_offset( *obj ) ); + fprintf( file, " offm, offn %lu, %lu\n", ( unsigned long int )bli_obj_row_off( *obj ), + ( unsigned long int )bli_obj_col_off( *obj ) ); fprintf( file, " diagoff %ld\n", ( signed long int )bli_obj_diag_offset( *obj ) ); fprintf( file, "\n" ); diff --git a/frame/base/bli_part.c b/frame/base/bli_part.c index b8a28d5415..64718353eb 100644 --- a/frame/base/bli_part.c +++ b/frame/base/bli_part.c @@ -592,6 +592,16 @@ void bli_acquire_mpart_tl2br( subpart_t requested_part, requested_part != BLIS_SUBPART11 && requested_part != BLIS_SUBPART22 ) { + // FGVZ: Fix me. This needs to be cleaned up. Either non-diagonal + // intersecting subpartitions should inherit their root object's + // uplo field, or it should not. Right now, they DO inherit the + // uplo (because they are not set to BLIS_DENSE when the diagonal + // does not intersect). But the whole point of being able to query + // the root object's properties (e.g. uplo field) was so that we + // COULD mark such subpartitions as dense, to make it easier for + // certain subproblems on those subpartitions--subproblems that + // are agnostic to where the subpartition came from. + // NOTE: This comment may be out-of-date since we now distinguish // between uplo properties for the current and root objects... // Note that we cannot mark the subpartition object as general/dense diff --git a/frame/base/bli_prune.c b/frame/base/bli_prune.c new file mode 100644 index 0000000000..d53049ecc3 --- /dev/null +++ b/frame/base/bli_prune.c @@ -0,0 +1,135 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + +void bli_prune_unref_mparts( obj_t* p, mdim_t mdim_p, + obj_t* s, mdim_t mdim_s ) +{ + // If the primary object is general, it has no structure, and + // therefore, no unreferenced parts. + if ( bli_obj_is_general( *p ) ) return; + + // If the primary object is BLIS_ZEROS, set the dimensions so that the + // matrix is empty. This is not strictly needed but rather a minor + // optimization, as it would prevent threads that would otherwise get + // subproblems on BLIS_ZEROS operands from calling the macro-kernel, + // because bli_get_range*() would return empty ranges, which would + // cause the variant's for loop from executing any iterations. + // NOTE: this should only ever execute if the primary object is + // triangular because that is the only structure type with subpartitions + // that can be marked as BLIS_ZEROS. + if ( bli_obj_is_triangular( *p ) && + bli_obj_is_zeros( *p ) ) { bli_obj_set_dim( mdim_p, 0, *p ); + bli_obj_set_dim( mdim_s, 0, *s ); + return; } + + // If the primary object is hermitian, symmetric, or triangular, we + // assume that the unstored region will be unreferenced (otherwise, + // the caller should not be invoking this function on that object). + //if ( bli_obj_is_herm_or_symm( *p ) || + // bli_obj_is_triangular( *p ) ) + { + doff_t diagoff_p = bli_obj_diag_offset( *p ); + dim_t m = bli_obj_length( *p ); + dim_t n = bli_obj_width( *p ); + uplo_t uplo = bli_obj_uplo( *p ); + dim_t off_inc = 0; + dim_t q; + + // Support implicit transposition on p and s. + if ( bli_obj_has_trans( *p ) ) + { + bli_reflect_about_diag( diagoff_p, uplo, m, n ); + bli_toggle_dim( mdim_p ); + } + if ( bli_obj_has_trans( *s ) ) + { + bli_toggle_dim( mdim_s ); + } + + // Prune away any zero region of the matrix depending on the + // dimension of the primary object being partitioned and the + // triangle in which it is stored. + if ( bli_obj_is_lower( *p ) ) + { + if ( bli_is_m_dim( mdim_p ) ) + { bli_prune_unstored_region_top_l( diagoff_p, m, n, off_inc ); } + else // if ( bli_is_n_dim( mdim_p ) ) + { bli_prune_unstored_region_right_l( diagoff_p, m, n, off_inc ); } + } + else if ( bli_obj_is_upper( *p ) ) + { + if ( bli_is_m_dim( mdim_p ) ) + { bli_prune_unstored_region_bottom_u( diagoff_p, m, n, off_inc ); } + else // if ( bli_is_n_dim( mdim_p ) ) + { bli_prune_unstored_region_left_u( diagoff_p, m, n, off_inc ); } + } + else if ( bli_obj_is_dense( *p ) ) + { + // Hermitian, symmetric, and triangular matrices are almost + // never dense, but if one were found to be dense, it would + // have no unreferenced regions to prune. + return; + } + else // if ( bli_obj_is_zeros( *p ) ) + { + // Sanity check. Hermitian/symmetric matrices should never have + // zero subpartitions. + bli_check_error_code( BLIS_NOT_YET_IMPLEMENTED ); + } + + // Select the (potentially modified) dimension along which we are + // partitioning. + if ( bli_is_m_dim( mdim_p ) ) q = m; + else /* if ( bli_is_n_dim( mdim_p ) ) */ q = n; + + // Update the affected objects in case anything changed. Notice that + // it is okay to update the dimension and diagonal offset fields of + // packed primary objects, as long as we do so in tandem with the + // secondary object to maintain conformality. This just means that + // the "ignore-able" zero region is skipped over here, rather than + // within the macro-kernel. + bli_obj_set_diag_offset( diagoff_p, *p ); + bli_obj_set_dim( mdim_p, q, *p ); + bli_obj_set_dim( mdim_s, q, *s ); + + // Only update the affected offset fields if the object in question + // is NOT a packed object. Otherwise, bli_obj_buffer_at_off() will + // compute the wrong address within the macro-kernel object wrapper. + if ( !bli_obj_is_packed( *p ) ) { bli_obj_inc_off( mdim_p, off_inc, *p ); } + if ( !bli_obj_is_packed( *s ) ) { bli_obj_inc_off( mdim_s, off_inc, *s ); } + } +} + diff --git a/frame/base/bli_prune.h b/frame/base/bli_prune.h new file mode 100644 index 0000000000..1f71610c80 --- /dev/null +++ b/frame/base/bli_prune.h @@ -0,0 +1,36 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +void bli_prune_unref_mparts( obj_t* p, mdim_t mdim_p, + obj_t* s, mdim_t mdim_s ); diff --git a/frame/base/bli_threading.c b/frame/base/bli_threading.c index 890a1d391e..077089dc97 100644 --- a/frame/base/bli_threading.c +++ b/frame/base/bli_threading.c @@ -157,16 +157,19 @@ void* bli_broadcast_structure( thread_comm_t* communicator, dim_t id, void* to_s } // Code for work assignments -void bli_get_range( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, bool_t handle_edge_low, dim_t* start, dim_t* end ) +void bli_get_range( void* thr, dim_t n, dim_t bf, bool_t handle_edge_low, dim_t* start, dim_t* end ) { thrinfo_t* thread = ( thrinfo_t* )thr; dim_t n_way = thread->n_way; dim_t work_id = thread->work_id; + dim_t all_start = 0; + dim_t all_end = n; + dim_t size = all_end - all_start; - dim_t n_bf_whole = size / block_factor; - dim_t n_bf_left = size % block_factor; + dim_t n_bf_whole = size / bf; + dim_t n_bf_left = size % bf; dim_t n_bf_lo = n_bf_whole / n_way; dim_t n_bf_hi = n_bf_whole / n_way; @@ -217,8 +220,8 @@ void bli_get_range( void* thr, dim_t all_start, dim_t all_end, dim_t block_facto // Compute the actual widths (in units of rows/columns) of // individual threads in the low and high groups. - dim_t size_lo = n_bf_lo * block_factor; - dim_t size_hi = n_bf_hi * block_factor; + dim_t size_lo = n_bf_lo * bf; + dim_t size_hi = n_bf_hi * bf; // Precompute the starting indices of the low and high groups. dim_t lo_start = all_start; @@ -257,8 +260,8 @@ void bli_get_range( void* thr, dim_t all_start, dim_t all_end, dim_t block_facto // Compute the actual widths (in units of rows/columns) of // individual threads in the low and high groups. - dim_t size_lo = n_bf_lo * block_factor; - dim_t size_hi = n_bf_hi * block_factor; + dim_t size_lo = n_bf_lo * bf; + dim_t size_hi = n_bf_hi * bf; // Precompute the starting indices of the low and high groups. dim_t lo_start = all_start; @@ -288,188 +291,514 @@ void bli_get_range( void* thr, dim_t all_start, dim_t all_end, dim_t block_facto } } -void bli_get_range_l2r( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, dim_t* start, dim_t* end ) +siz_t bli_get_range_l2r( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ) { - bli_get_range( thr, all_start, all_end, block_factor, + dim_t m = bli_obj_length_after_trans( *a ); + dim_t n = bli_obj_width_after_trans( *a ); + + bli_get_range( thr, n, bf, FALSE, start, end ); + + return m * ( *end - *start ); } -void bli_get_range_r2l( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, dim_t* start, dim_t* end ) +siz_t bli_get_range_r2l( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ) { - bli_get_range( thr, all_start, all_end, block_factor, + dim_t m = bli_obj_length_after_trans( *a ); + dim_t n = bli_obj_width_after_trans( *a ); + + bli_get_range( thr, n, bf, TRUE, start, end ); + + return m * ( *end - *start ); } -void bli_get_range_t2b( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, dim_t* start, dim_t* end ) +siz_t bli_get_range_t2b( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ) { - bli_get_range( thr, all_start, all_end, block_factor, + dim_t m = bli_obj_length_after_trans( *a ); + dim_t n = bli_obj_width_after_trans( *a ); + + bli_get_range( thr, m, bf, FALSE, start, end ); + + return n * ( *end - *start ); } -void bli_get_range_b2t( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, dim_t* start, dim_t* end ) +siz_t bli_get_range_b2t( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ) { - bli_get_range( thr, all_start, all_end, block_factor, + dim_t m = bli_obj_length_after_trans( *a ); + dim_t n = bli_obj_width_after_trans( *a ); + + bli_get_range( thr, m, bf, TRUE, start, end ); + + return n * ( *end - *start ); +} + +dim_t bli_get_range_width_l( doff_t diagoff_j, + dim_t m, + dim_t n_j, + dim_t j, + dim_t n_way, + dim_t bf, + dim_t bf_left, + double area_per_thr, + bool_t handle_edge_low ) +{ + dim_t width; + + // In this function, we assume that we are somewhere in the process of + // partitioning an m x n lower-stored region (with arbitrary diagonal + // offset) n_ways along the n dimension (into column panels). The value + // j identifies the left-to-right subpartition index (from 0 to n_way-1) + // of the subpartition whose width we are about to compute using the + // area per thread determined by the caller. n_j is the number of + // columns in the remaining region of the matrix being partitioned, + // and diagoff_j is that region's diagonal offset. + + // If this is the last subpartition, the width is simply equal to n_j. + // Note that this statement handles cases where the "edge case" (if + // one exists) is assigned to the high end of the index range (ie: + // handle_edge_low == FALSE). + if ( j == n_way - 1 ) return n_j; + + // At this point, we know there are at least two subpartitions left. + // We also know that IF the submatrix contains a completely dense + // rectangular submatrix, it will occur BEFORE the triangular (or + // trapezoidal) part. + + // Here, we implement a somewhat minor load balancing optimization + // that ends up getting employed only for relatively small matrices. + // First, recall that all subpartition widths will be some multiple + // of the blocking factor bf, except perhaps either the first or last + // subpartition, which will receive the edge case, if it exists. + // Also recall that j represents the current thread (or thread group, + // or "caucus") for which we are computing a subpartition width. + // If n_j is sufficiently small that we can only allocate bf columns + // to each of the remaining threads, then we set the width to bf. We + // do not allow the subpartition width to be less than bf, so, under + // some conditions, if n_j is small enough, some of the reamining + // threads may not get any work. For the purposes of this lower bound + // on work (ie: width >= bf), we allow the edge case to count as a + // "full" set of bf columns. + { + dim_t n_j_bf = n_j / bf + ( bf_left > 0 ? 1 : 0 ); + + if ( n_j_bf <= n_way - j ) + { + if ( j == 0 && handle_edge_low ) + width = ( bf_left > 0 ? bf_left : bf ); + else + width = bf; + + // Make sure that the width does not exceed n_j. This would + // occur if and when n_j_bf < n_way - j; that is, when the + // matrix being partitioned is sufficiently small relative to + // n_way such that there is not even enough work for every + // (remaining) thread to get bf (or bf_left) columns. The + // net effect of this safeguard is that some threads may get + // assigned empty ranges (ie: no work), which of course must + // happen in some situations. + if ( width > n_j ) width = n_j; + + return width; + } + } + + // This block computes the width assuming that we are entirely within + // a dense rectangle that precedes the triangular (or trapezoidal) + // part. + { + // First compute the width of the current panel under the + // assumption that the diagonal offset would not intersect. + width = ( dim_t )bli_round( ( double )area_per_thr / ( double )m ); + + // Adjust the width, if necessary. Specifically, we may need + // to allocate the edge case to the first subpartition, if + // requested; otherwise, we just need to ensure that the + // subpartition is a multiple of the blocking factor. + if ( j == 0 && handle_edge_low ) + { + if ( width % bf != bf_left ) width += bf_left - ( width % bf ); + } + else // if interior case + { + // Round up to the next multiple of the blocking factor. + //if ( width % bf != 0 ) width += bf - ( width % bf ); + // Round to the nearest multiple of the blocking factor. + if ( width % bf != 0 ) width = bli_round_to_mult( width, bf ); + } + } + + // We need to recompute width if the panel, according to the width + // as currently computed, would intersect the diagonal. + if ( diagoff_j < width ) + { + dim_t offm_inc, offn_inc; + + // Prune away the unstored region above the diagonal, if it exists. + // Note that the entire region was pruned initially, so we know that + // we don't need to try to prune the right side. (Also, we discard + // the offset deltas since we don't need to actually index into the + // subpartition.) + bli_prune_unstored_region_top_l( diagoff_j, m, n_j, offm_inc ); + //bli_prune_unstored_region_right_l( diagoff_j, m, n_j, offn_inc ); + + // We don't need offm_inc, offn_inc here. These statements should + // prevent compiler warnings. + ( void )offm_inc; + ( void )offn_inc; + + // Solve a quadratic equation to find the width of the current (jth) + // subpartition given the m dimension, diagonal offset, and area. + // NOTE: We know that the +/- in the quadratic formula must be a + + // here because we know that the desired solution (the subpartition + // width) will be smaller than (m + diagoff), not larger. If you + // don't believe me, draw a picture! + const double a = -0.5; + const double b = ( double )m + ( double )diagoff_j + 0.5; + const double c = -0.5 * ( ( double )diagoff_j * + ( ( double )diagoff_j + 1.0 ) + ) - area_per_thr; + const double x = ( -b + sqrt( b * b - 4.0 * a * c ) ) / ( 2.0 * a ); + + // Use the rounded solution as our width, but make sure it didn't + // round to zero. + width = ( dim_t )bli_round( x ); + if ( width == 0 ) width = 1; + + // Adjust the width, if necessary. + if ( j == 0 && handle_edge_low ) + { + if ( width % bf != bf_left ) width += bf_left - ( width % bf ); + } + else // if interior case + { + // Round up to the next multiple of the blocking factor. + //if ( width % bf != 0 ) width += bf - ( width % bf ); + // Round to the nearest multiple of the blocking factor. + if ( width % bf != 0 ) width = bli_round_to_mult( width, bf ); + } + } + + // Make sure that the width, after being adjusted, does not cause the + // subpartition to exceed n_j. + if ( width > n_j ) width = n_j; + + return width; } -void bli_get_range_weighted( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, uplo_t uplo, bool_t handle_edge_low, dim_t* start, dim_t* end ) +siz_t bli_find_area_trap_l( dim_t m, dim_t n, doff_t diagoff ) +{ + dim_t offm_inc = 0; + dim_t offn_inc = 0; + double tri_area; + double area; + + // Prune away any rectangular region above where the diagonal + // intersects the left edge of the subpartition, if it exists. + bli_prune_unstored_region_top_l( diagoff, m, n, offm_inc ); + + // Prune away any rectangular region to the right of where the + // diagonal intersects the bottom edge of the subpartition, if + // it exists. (This shouldn't ever be needed, since the caller + // would presumably have already performed rightward pruning, + // but it's here just in case.) + bli_prune_unstored_region_right_l( diagoff, m, n, offn_inc ); + + ( void )offm_inc; + ( void )offn_inc; + + // Compute the area of the empty triangle so we can subtract it + // from the area of the rectangle that bounds the subpartition. + if ( bli_intersects_diag_n( diagoff, m, n ) ) + { + double tri_dim = ( double )( n - diagoff - 1 ); + tri_area = tri_dim * ( tri_dim + 1.0 ) / 2.0; + } + else + { + // If the diagonal does not intersect the trapezoid, then + // we can compute the area as a simple rectangle. + tri_area = 0.0; + } + + area = ( double )m * ( double )n - tri_area; + + return ( siz_t )area; +} + +siz_t bli_get_range_weighted( void* thr, + doff_t diagoff, + uplo_t uplo, + dim_t m, + dim_t n, + dim_t bf, + bool_t handle_edge_low, + dim_t* j_start_thr, + dim_t* j_end_thr ) { thrinfo_t* thread = ( thrinfo_t* )thr; + dim_t n_way = thread->n_way; - dim_t work_id = thread->work_id; - dim_t size = all_end - all_start; - dim_t width; - dim_t block_fac_leftover = size % block_factor; - dim_t i; - double num; + dim_t my_id = thread->work_id; + + dim_t bf_left = n % bf; + + dim_t j; + + dim_t off_j; + doff_t diagoff_j; + dim_t n_left; + + dim_t width_j; - *start = 0; - *end = all_end - all_start; - num = size * size / ( double )n_way; + dim_t offm_inc, offn_inc; + + double tri_dim, tri_area; + double area_total, area_per_thr; + + siz_t area = 0; + + // In this function, we assume that the caller has already determined + // that (a) the diagonal intersects the submatrix, and (b) the submatrix + // is either lower- or upper-stored. if ( bli_is_lower( uplo ) ) { - dim_t cur_caucus = n_way - 1; - dim_t len = 0; - - // This loop computes subpartitions backwards, from the high end - // of the index range to the low end. If the low end is assumed - // to be on the left and the high end the right, this assignment - // of widths is appropriate for n dimension partitioning of a - // lower triangular matrix. - for ( i = 0; TRUE; ++i ) + // Prune away the unstored region above the diagonal, if it exists, + // and then to the right of where the diagonal intersects the bottom, + // if it exists. (Also, we discard the offset deltas since we don't + // need to actually index into the subpartition.) + bli_prune_unstored_region_top_l( diagoff, m, n, offm_inc ); + bli_prune_unstored_region_right_l( diagoff, m, n, offn_inc ); + + // We don't need offm_inc, offn_inc here. These statements should + // prevent compiler warnings. + ( void )offm_inc; + ( void )offn_inc; + + // Now that pruning has taken place, we know that diagoff >= 0. + + // Compute the total area of the submatrix, accounting for the + // location of the diagonal, and divide it by the number of ways + // of parallelism. + tri_dim = ( double )( n - diagoff - 1 ); + tri_area = tri_dim * ( tri_dim + 1.0 ) / 2.0; + area_total = ( double )m * ( double )n - tri_area; + area_per_thr = area_total / ( double )n_way; + + // Initialize some variables prior to the loop: the offset to the + // current subpartition, the remainder of the n dimension, and + // the diagonal offset of the current subpartition. + off_j = 0; + diagoff_j = diagoff; + n_left = n; + + // Iterate over the subpartition indices corresponding to each + // thread/caucus participating in the n_way parallelism. + for ( j = 0; j < n_way; ++j ) { - width = ceil( sqrt( len*len + num ) ) - len; - - // If we need to allocate the edge case (assuming it exists) - // to the high thread subpartition, adjust width so that it - // contains the exact amount of leftover edge dimension so that - // all remaining subpartitions can be multiples of block_factor. - // If the edge case is to be allocated to the low subpartition, - // or if there is no edge case, it is implicitly allocated to - // the low subpartition by virtue of the fact that all other - // subpartitions already assigned will be multiples of - // block_factor. - if ( i == 0 && !handle_edge_low ) - { - if ( width % block_factor != block_fac_leftover ) - width += block_fac_leftover - ( width % block_factor ); - } - else + // Compute the width of the jth subpartition, taking the + // current diagonal offset into account, if needed. + width_j = bli_get_range_width_l( diagoff_j, m, n_left, + j, n_way, + bf, bf_left, + area_per_thr, + handle_edge_low ); + + // If the current thread belongs to caucus j, this is his + // subpartition. So we compute the implied index range and + // end our search. + if ( j == my_id ) { - if ( width % block_factor != 0 ) - width += block_factor - ( width % block_factor ); - } + *j_start_thr = off_j; + *j_end_thr = off_j + width_j; - if ( cur_caucus == work_id ) - { - *start = bli_max( 0, *end - width ) + all_start; - *end = *end + all_start; - return; - } - else - { - *end -= width; - len += width; - cur_caucus--; + area = bli_find_area_trap_l( m, width_j, diagoff_j ); + + break; } + + // Shift the current subpartition's starting and diagonal offsets, + // as well as the remainder of the n dimension, according to the + // computed width, and then iterate to the next subpartition. + off_j += width_j; + diagoff_j -= width_j; + n_left -= width_j; } } else // if ( bli_is_upper( uplo ) ) { - // This loop computes subpartitions forwards, from the low end - // of the index range to the high end. If the low end is assumed - // to be on the left and the high end the right, this assignment - // of widths is appropriate for n dimension partitioning of an - // upper triangular matrix. - for ( i = 0; TRUE; ++i ) - { - width = ceil( sqrt( *start * *start + num ) ) - *start; - - if ( i == 0 && handle_edge_low ) - { - if ( width % block_factor != block_fac_leftover ) - width += block_fac_leftover - ( width % block_factor ); - } - else - { - if ( width % block_factor != 0 ) - width += block_factor - ( width % block_factor ); - } - - if ( work_id == 0 ) - { - *start = *start + all_start; - *end = bli_min( *start + width, all_end ); - return; - } - else - { - *start = *start + width; - work_id--; - } - } + // Express the upper-stored case in terms of the lower-stored case. + + // First, we convert the upper-stored trapezoid to an equivalent + // lower-stored trapezoid by rotating it 180 degrees. + bli_rotate180_trapezoid( diagoff, uplo ); + + // Now that the trapezoid is "flipped" in the n dimension, negate + // the bool that encodes whether to handle the edge case at the + // low (or high) end of the index range. + bli_toggle_bool( handle_edge_low ); + + // Compute the appropriate range for the rotated trapezoid. + area = bli_get_range_weighted( thr, diagoff, uplo, m, n, bf, + handle_edge_low, + j_start_thr, j_end_thr ); + + // Reverse the indexing basis for the subpartition ranges so that + // the indices, relative to left-to-right iteration through the + // unrotated upper-stored trapezoid, map to the correct columns + // (relative to the diagonal). This amounts to subtracting the + // range from n. + bli_reverse_index_direction( *j_start_thr, *j_end_thr, n ); } + + return area; } -void bli_get_range_weighted_l2r( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, uplo_t uplo, dim_t* start, dim_t* end ) +siz_t bli_get_range_weighted_l2r( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ) { - if ( bli_is_upper_or_lower( uplo ) ) + siz_t area; + + // This function assigns area-weighted ranges in the n dimension + // where the total range spans 0 to n-1 with 0 at the left end and + // n-1 at the right end. + + if ( bli_obj_intersects_diag( *a ) && + bli_obj_is_upper_or_lower( *a ) ) { - bli_get_range_weighted( thr, all_start, all_end, block_factor, - uplo, FALSE, start, end ); + doff_t diagoff = bli_obj_diag_offset( *a ); + uplo_t uplo = bli_obj_uplo( *a ); + dim_t m = bli_obj_length( *a ); + dim_t n = bli_obj_width( *a ); + + // Support implicit transposition. + if ( bli_obj_has_trans( *a ) ) + { + bli_reflect_about_diag( diagoff, uplo, m, n ); + } + + area = bli_get_range_weighted( thr, diagoff, uplo, m, n, bf, + FALSE, start, end ); } else // if dense or zeros { - bli_get_range_l2r( thr, all_start, all_end, block_factor, - start, end ); + area = bli_get_range_l2r( thr, a, bf, + start, end ); } + + return area; } -void bli_get_range_weighted_r2l( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, uplo_t uplo, dim_t* start, dim_t* end ) +siz_t bli_get_range_weighted_r2l( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ) { - if ( bli_is_upper_or_lower( uplo ) ) + siz_t area; + + // This function assigns area-weighted ranges in the n dimension + // where the total range spans 0 to n-1 with 0 at the right end and + // n-1 at the left end. + + if ( bli_obj_intersects_diag( *a ) && + bli_obj_is_upper_or_lower( *a ) ) { -//printf( "bli_get_range_weighted_r2l: is upper or lower\n" ); - bli_toggle_uplo( uplo ); - bli_get_range_weighted( thr, all_start, all_end, block_factor, - uplo, TRUE, start, end ); + doff_t diagoff = bli_obj_diag_offset( *a ); + uplo_t uplo = bli_obj_uplo( *a ); + dim_t m = bli_obj_length( *a ); + dim_t n = bli_obj_width( *a ); + + // Support implicit transposition. + if ( bli_obj_has_trans( *a ) ) + { + bli_reflect_about_diag( diagoff, uplo, m, n ); + } + + bli_rotate180_trapezoid( diagoff, uplo ); + + area = bli_get_range_weighted( thr, diagoff, uplo, m, n, bf, + TRUE, start, end ); } else // if dense or zeros { -//printf( "bli_get_range_weighted_r2l: is dense or zeros\n" ); - bli_get_range_r2l( thr, all_start, all_end, block_factor, - start, end ); + area = bli_get_range_r2l( thr, a, bf, + start, end ); } + + return area; } -void bli_get_range_weighted_t2b( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, uplo_t uplo, dim_t* start, dim_t* end ) +siz_t bli_get_range_weighted_t2b( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ) { - if ( bli_is_upper_or_lower( uplo ) ) + siz_t area; + + // This function assigns area-weighted ranges in the m dimension + // where the total range spans 0 to m-1 with 0 at the top end and + // m-1 at the bottom end. + + if ( bli_obj_intersects_diag( *a ) && + bli_obj_is_upper_or_lower( *a ) ) { - bli_toggle_uplo( uplo ); - bli_get_range_weighted( thr, all_start, all_end, block_factor, - uplo, FALSE, start, end ); + doff_t diagoff = bli_obj_diag_offset( *a ); + uplo_t uplo = bli_obj_uplo( *a ); + dim_t m = bli_obj_length( *a ); + dim_t n = bli_obj_width( *a ); + + // Support implicit transposition. + if ( bli_obj_has_trans( *a ) ) + { + bli_reflect_about_diag( diagoff, uplo, m, n ); + } + + bli_reflect_about_diag( diagoff, uplo, m, n ); + + area = bli_get_range_weighted( thr, diagoff, uplo, m, n, bf, + FALSE, start, end ); } else // if dense or zeros { - bli_get_range_t2b( thr, all_start, all_end, block_factor, - start, end ); + area = bli_get_range_t2b( thr, a, bf, + start, end ); } + + return area; } -void bli_get_range_weighted_b2t( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, uplo_t uplo, dim_t* start, dim_t* end ) +siz_t bli_get_range_weighted_b2t( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ) { - if ( bli_is_upper_or_lower( uplo ) ) + siz_t area; + + // This function assigns area-weighted ranges in the m dimension + // where the total range spans 0 to m-1 with 0 at the bottom end and + // m-1 at the top end. + + if ( bli_obj_intersects_diag( *a ) && + bli_obj_is_upper_or_lower( *a ) ) { - bli_get_range_weighted( thr, all_start, all_end, block_factor, - uplo, TRUE, start, end ); + doff_t diagoff = bli_obj_diag_offset( *a ); + uplo_t uplo = bli_obj_uplo( *a ); + dim_t m = bli_obj_length( *a ); + dim_t n = bli_obj_width( *a ); + + // Support implicit transposition. + if ( bli_obj_has_trans( *a ) ) + { + bli_reflect_about_diag( diagoff, uplo, m, n ); + } + + bli_reflect_about_diag( diagoff, uplo, m, n ); + + bli_rotate180_trapezoid( diagoff, uplo ); + + area = bli_get_range_weighted( thr, diagoff, uplo, m, n, bf, + TRUE, start, end ); } else // if dense or zeros { - bli_get_range_b2t( thr, all_start, all_end, block_factor, - start, end ); + area = bli_get_range_b2t( thr, a, bf, + start, end ); } + + return area; } diff --git a/frame/base/bli_threading.h b/frame/base/bli_threading.h index 89b4384748..5fdff80fd0 100644 --- a/frame/base/bli_threading.h +++ b/frame/base/bli_threading.h @@ -101,13 +101,25 @@ void bli_barrier( thread_comm_t* communicator, dim_t thread_id ); struct thrinfo_s { - thread_comm_t* ocomm; //The thread communicator for the other threads sharing the same work at this level - dim_t ocomm_id; //Our thread id within that thread comm - thread_comm_t* icomm; //The thread communicator for the other threads sharing the same work at this level - dim_t icomm_id; //Our thread id within that thread comm + // The thread communicator for the other threads sharing the same work + // at this level. + thread_comm_t* ocomm; - dim_t n_way; //Number of distinct used to parallelize the loop - dim_t work_id; //What we're working on + // Our thread id within the ocomm thread communicator. + dim_t ocomm_id; + + // The thread communicator for the other threads sharing the same work + // at this level. + thread_comm_t* icomm; + + // Our thread id within the icomm thread communicator. + dim_t icomm_id; + + // The number of distinct threads used to parallelize the loop. + dim_t n_way; + + // What we're working on. + dim_t work_id; }; typedef struct thrinfo_s thrinfo_t; @@ -128,39 +140,37 @@ typedef struct thrinfo_s thrinfo_t; #define thread_obarrier( thread ) bli_barrier( thread->ocomm, thread->ocomm_id ) #define thread_ibarrier( thread ) bli_barrier( thread->icomm, thread->icomm_id ) -void bli_get_range( void* thr, dim_t all_start, dim_t all_end, - dim_t block_factor, - bool_t handle_edge_low, +void bli_get_range( void* thr, dim_t n, dim_t bf, bool_t handle_edge_low, dim_t* start, dim_t* end ); -void bli_get_range_l2r( void* thr, dim_t all_start, dim_t all_end, - dim_t block_factor, - dim_t* start, dim_t* end ); -void bli_get_range_r2l( void* thr, dim_t all_start, dim_t all_end, - dim_t block_factor, - dim_t* start, dim_t* end ); -void bli_get_range_t2b( void* thr, dim_t all_start, dim_t all_end, - dim_t block_factor, - dim_t* start, dim_t* end ); -void bli_get_range_b2t( void* thr, dim_t all_start, dim_t all_end, - dim_t block_factor, - dim_t* start, dim_t* end ); - -void bli_get_range_weighted( void* thr, dim_t all_start, dim_t all_end, - dim_t block_factor, uplo_t uplo, - bool_t handle_edge_low, - dim_t* start, dim_t* end ); -void bli_get_range_weighted_l2r( void* thr, dim_t all_start, dim_t all_end, - dim_t block_factor, uplo_t uplo, - dim_t* start, dim_t* end ); -void bli_get_range_weighted_r2l( void* thr, dim_t all_start, dim_t all_end, - dim_t block_factor, uplo_t uplo, - dim_t* start, dim_t* end ); -void bli_get_range_weighted_t2b( void* thr, dim_t all_start, dim_t all_end, - dim_t block_factor, uplo_t uplo, - dim_t* start, dim_t* end ); -void bli_get_range_weighted_b2t( void* thr, dim_t all_start, dim_t all_end, - dim_t block_factor, uplo_t uplo, - dim_t* start, dim_t* end ); +siz_t bli_get_range_l2r( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ); +siz_t bli_get_range_r2l( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ); +siz_t bli_get_range_t2b( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ); +siz_t bli_get_range_b2t( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ); + +dim_t bli_get_range_width_l( doff_t diagoff_j, + dim_t m, + dim_t n_j, + dim_t j, + dim_t n_way, + dim_t bf, + dim_t bf_left, + double area_per_thr, + bool_t handle_edge_low ); +siz_t bli_find_area_trap_l( dim_t m, dim_t n, doff_t diagoff ); +siz_t bli_get_range_weighted( void* thr, + doff_t diagoff, + uplo_t uplo, + dim_t m, + dim_t n, + dim_t bf, + bool_t handle_edge_low, + dim_t* j_start_thr, + dim_t* j_end_thr ); + +siz_t bli_get_range_weighted_l2r( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ); +siz_t bli_get_range_weighted_r2l( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ); +siz_t bli_get_range_weighted_t2b( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ); +siz_t bli_get_range_weighted_b2t( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ); thrinfo_t* bli_create_thread_info( thread_comm_t* ocomm, dim_t ocomm_id, thread_comm_t* icomm, dim_t icomm_id, diff --git a/frame/include/bli_obj_macro_defs.h b/frame/include/bli_obj_macro_defs.h index 10bb01f5c2..5f126c987f 100644 --- a/frame/include/bli_obj_macro_defs.h +++ b/frame/include/bli_obj_macro_defs.h @@ -156,8 +156,8 @@ #define bli_obj_is_upper_or_lower( obj ) \ \ - ( ( (obj).info & BLIS_UPLO_BITS ) == BLIS_BITVAL_UPPER || \ - ( (obj).info & BLIS_UPLO_BITS ) == BLIS_BITVAL_LOWER ) + ( bli_obj_is_upper( obj ) || \ + bli_obj_is_lower( obj ) ) #define bli_obj_is_dense( obj ) \ \ @@ -441,11 +441,15 @@ #define bli_obj_length( obj ) \ \ - ((obj).m) + ( (obj).dim[BLIS_M] ) #define bli_obj_width( obj ) \ \ - ((obj).n) + ( (obj).dim[BLIS_N] ) + +#define bli_obj_dim( mdim, obj ) \ +\ + ( (obj).dim[mdim] ) #define bli_obj_min_dim( obj ) \ \ @@ -579,23 +583,38 @@ bli_obj_width_stored( obj ) // Dimension modification +#define bli_obj_set_length( dim_m, obj ) \ +{ \ + (obj).dim[BLIS_M] = dim_m; \ +} + +#define bli_obj_set_width( dim_n, obj ) \ +{ \ + (obj).dim[BLIS_N] = dim_n; \ +} + +#define bli_obj_set_dim( mdim, dim_val, obj ) \ +{ \ + (obj).dim[mdim] = dim_val; \ +} + #define bli_obj_set_dims( dim_m, dim_n, obj ) \ { \ - (obj).m = dim_m; \ - (obj).n = dim_n; \ + bli_obj_set_length( dim_m, obj ); \ + bli_obj_set_width( dim_n, obj ); \ } #define bli_obj_set_dims_with_trans( trans, dim_m, dim_n, obj ) \ { \ if ( bli_does_notrans( trans ) ) \ { \ - (obj).m = dim_m; \ - (obj).n = dim_n; \ + bli_obj_set_length( dim_m, obj ); \ + bli_obj_set_width( dim_n, obj ); \ } \ else \ { \ - (obj).m = dim_n; \ - (obj).n = dim_m; \ + bli_obj_set_length( dim_n, obj ); \ + bli_obj_set_width( dim_m, obj ); \ } \ } @@ -604,15 +623,15 @@ bli_obj_width_stored( obj ) #define bli_obj_row_stride( obj ) \ \ - ((obj).rs) + ( (obj).rs ) #define bli_obj_col_stride( obj ) \ \ - ((obj).cs) + ( (obj).cs ) #define bli_obj_imag_stride( obj ) \ \ - ((obj).is) + ( (obj).is ) #define bli_obj_row_stride_mag( obj ) \ \ @@ -671,41 +690,60 @@ bli_obj_width_stored( obj ) // Offset query -#define bli_obj_row_offset( obj ) \ +#define bli_obj_row_off( obj ) \ +\ + ( (obj).off[BLIS_M] ) + +#define bli_obj_col_off( obj ) \ \ - ( (obj).offm ) + ( (obj).off[BLIS_N] ) -#define bli_obj_col_offset( obj ) \ +#define bli_obj_off( mdim, obj ) \ \ - ( (obj).offn ) + ( (obj).off[mdim] ) // Offset modification +#define bli_obj_set_off( mdim, offset, obj ) \ +{ \ + (obj).off[mdim] = offset; \ +} + #define bli_obj_set_offs( offset_m, offset_n, obj ) \ { \ - (obj).offm = offset_m; \ - (obj).offn = offset_n; \ + bli_obj_set_off( BLIS_M, offset_m, obj ); \ + bli_obj_set_off( BLIS_N, offset_n, obj ); \ } -#define bli_obj_inc_offs( offset_m, offset_n, obj ) \ +#define bli_obj_inc_off( mdim, offset, obj ) \ +{ \ + (obj).off[mdim] += offset; \ +} + +#define bli_obj_inc_offm( offset, obj ) \ +{ \ + bli_obj_inc_off( BLIS_M, offset, obj ); \ +} + +#define bli_obj_inc_offn( offset, obj ) \ { \ - (obj).offm += offset_m; \ - (obj).offn += offset_n; \ + bli_obj_inc_off( BLIS_N, offset, obj ); \ } -#define bli_obj_dec_offs( offset_m, offset_n, obj ) \ +#define bli_obj_inc_offs( offset_m, offset_n, obj ) \ { \ - (obj).offm -= offset_m; \ - (obj).offn -= offset_n; \ + bli_obj_inc_off( BLIS_M, offset_m, obj ); \ + bli_obj_inc_off( BLIS_N, offset_n, obj ); \ } + // Diagonal offset query #define bli_obj_diag_offset( obj ) \ \ - ((obj).diag_off) + ( (obj).diag_off ) #define bli_obj_diag_offset_after_trans( obj ) \ \ @@ -762,7 +800,7 @@ bli_obj_width_stored( obj ) #define bli_obj_buffer( obj ) \ \ - (obj).buffer + ( (obj).buffer ) // Buffer address modification @@ -776,7 +814,7 @@ bli_obj_width_stored( obj ) #define bli_obj_internal_scalar_buffer( obj ) \ \ - &((obj).scalar) + &( (obj).scalar ) // Bufferless scalar field modification @@ -794,7 +832,7 @@ bli_obj_width_stored( obj ) #define bli_obj_elem_size( obj ) \ \ - (obj).elem_size \ + ( (obj).elem_size ) // Element size modification @@ -851,19 +889,19 @@ bli_obj_width_stored( obj ) #define bli_obj_panel_length( obj ) \ \ - ((obj).m_panel) + ( (obj).m_panel ) #define bli_obj_panel_width( obj ) \ \ - ((obj).n_panel) + ( (obj).n_panel ) #define bli_obj_panel_dim( obj ) \ \ - ((obj).pd) + ( (obj).pd ) #define bli_obj_panel_stride( obj ) \ \ - ((obj).ps) + ( (obj).ps ) // Packed panel info modification @@ -969,15 +1007,19 @@ bli_obj_width_stored( obj ) #define bli_obj_buffer_for_const( dt, obj ) \ \ ( void* )( \ - ( ( char* )( (obj).buffer ) ) + dt * BLIS_CONSTANT_SLOT_SIZE \ + ( ( char* )( bli_obj_buffer( obj ) ) ) + \ + ( dim_t )( dt * BLIS_CONSTANT_SLOT_SIZE ) \ ) #define bli_obj_buffer_at_off( obj ) \ \ ( void* )( \ - ( ( char* )( (obj).buffer ) ) + ( dim_t )(obj).elem_size * \ - ( (obj).offn * (obj).cs + \ - (obj).offm * (obj).rs ) \ + ( ( char* )( bli_obj_buffer ( obj ) ) + \ + ( dim_t )( bli_obj_elem_size( obj ) ) * \ + ( bli_obj_col_off( obj ) * bli_obj_col_stride( obj ) + \ + bli_obj_row_off( obj ) * bli_obj_row_stride( obj ) \ + ) \ + ) \ ) #define bli_obj_buffer_for_1x1( dt, obj ) \ @@ -1015,8 +1057,8 @@ bli_obj_width_stored( obj ) dim_t n = bli_obj_width( obj ); \ inc_t rs = bli_obj_row_stride( obj ); \ inc_t cs = bli_obj_col_stride( obj ); \ - dim_t offm = bli_obj_row_offset( obj ); \ - dim_t offn = bli_obj_col_offset( obj ); \ + dim_t offm = bli_obj_row_off( obj ); \ + dim_t offn = bli_obj_col_off( obj ); \ doff_t diag_off = bli_obj_diag_offset( obj ); \ \ bli_obj_set_dims( n, m, obj ); \ @@ -1047,8 +1089,8 @@ bli_obj_width_stored( obj ) { \ dim_t m = bli_obj_length( obj ); \ dim_t n = bli_obj_width( obj ); \ - dim_t offm = bli_obj_row_offset( obj ); \ - dim_t offn = bli_obj_col_offset( obj ); \ + dim_t offm = bli_obj_row_off( obj ); \ + dim_t offn = bli_obj_col_off( obj ); \ doff_t diag_off = bli_obj_diag_offset( obj ); \ \ bli_obj_set_dims( n, m, obj ); \ diff --git a/frame/include/bli_param_macro_defs.h b/frame/include/bli_param_macro_defs.h index 40a054ffa9..1b1040b1fb 100644 --- a/frame/include/bli_param_macro_defs.h +++ b/frame/include/bli_param_macro_defs.h @@ -144,7 +144,8 @@ #define bli_is_upper_or_lower( uplo ) \ \ - ( bli_is_upper( uplo ) || bli_is_lower( uplo ) ) + ( bli_is_upper( uplo ) || \ + bli_is_lower( uplo ) ) #define bli_is_dense( uplo ) \ \ @@ -470,6 +471,106 @@ ( bli_is_lower( uplo ) && bli_is_strictly_above_diag_n( diagoff, m, n ) ) ) +// pruning-related + +#define bli_prune_unstored_region_top_l( diagoff, m, n, offm_inc ) \ +{ \ + offm_inc = 0; \ +\ + /* If the diagonal intersects the left side of the matrix, + ignore the area above that intersection. */ \ + if ( diagoff < 0 ) \ + { \ + m = m + diagoff; \ + offm_inc = - diagoff; \ + diagoff = 0; \ + } \ +} + +#define bli_prune_unstored_region_right_l( diagoff, m, n, offn_inc ) \ +{ \ + offn_inc = 0; \ +\ + /* If the diagonal intersects the bottom side of the matrix, + ignore the area to the right of that intersection. */ \ + if ( n > diagoff + m ) \ + { \ + n = diagoff + m; \ + } \ +} + +#define bli_prune_unstored_region_left_u( diagoff, m, n, offn_inc ) \ +{ \ + offn_inc = 0; \ +\ + /* If the diagonal intersects the top side of the matrix, + ignore the area to the left of that intersection. */ \ + if ( diagoff > 0 ) \ + { \ + n = n - diagoff; \ + offn_inc = + diagoff; \ + diagoff = 0; \ + } \ +} + +#define bli_prune_unstored_region_bottom_u( diagoff, m, n, offm_inc ) \ +{ \ + offm_inc = 0; \ +\ + /* If the diagonal intersects the right side of the matrix, + ignore the area below that intersection. */ \ + if ( m > -diagoff + n ) \ + { \ + m = -diagoff + n; \ + } \ +} + + +// thread range-related + +#define bli_rotate180_trapezoid( diagoff, uplo ) \ +{ \ + diagoff = n - diagoff - m; \ + bli_toggle_uplo( uplo ); \ +} + +#define bli_reverse_index_direction( start, end, n ) \ +{ \ + dim_t start2 = n - start; \ + dim_t end2 = n - end; \ + start = end2; \ + end = start2; \ +} + +#define bli_reflect_about_diag( diagoff, uplo, m, n ) \ +{ \ + bli_swap_dims( m, n ); \ + bli_negate_diag_offset( diagoff ); \ + bli_toggle_uplo( uplo ); \ +} + + +// mdim_t-related + +#define bli_is_m_dim( mdim ) \ +\ + ( mdim == BLIS_M ) + +#define bli_is_n_dim( mdim ) \ +\ + ( mdim == BLIS_N ) + +#define bli_dim_toggled( mdim ) \ +\ + ( mdim == BLIS_M ? BLIS_N : BLIS_M ) + +#define bli_toggle_dim( mdim ) \ +{ \ + mdim = bli_dim_toggled( mdim ); \ +} + + + // index-related #define bli_is_edge_f( i1, iter, left ) \ diff --git a/frame/include/bli_scalar_macro_defs.h b/frame/include/bli_scalar_macro_defs.h index 6669c0ab94..32258e86e3 100644 --- a/frame/include/bli_scalar_macro_defs.h +++ b/frame/include/bli_scalar_macro_defs.h @@ -243,6 +243,22 @@ bli_fmax( bli_fabs( a ), \ bli_fabs( b ) ) +// round + +#define bli_round( val ) \ +\ + ( round( val ) ) + +// round_to_mult + +#define bli_round_to_mult( val, mult ) \ +\ + ( guint_t )( ( ( ( guint_t )val + \ + ( guint_t )mult / 2 \ + ) / mult \ + ) * mult \ + ) + // isnan, isinf #define bli_isinf( a ) isinf( a ) diff --git a/frame/include/bli_type_defs.h b/frame/include/bli_type_defs.h index a8f5bcb5b1..95b7c6c0e6 100644 --- a/frame/include/bli_type_defs.h +++ b/frame/include/bli_type_defs.h @@ -591,10 +591,8 @@ typedef struct obj_s // Basic fields struct obj_s* root; - dim_t offm; - dim_t offn; - dim_t m; - dim_t n; + dim_t off[2]; + dim_t dim[2]; doff_t diag_off; objbits_t info; @@ -626,10 +624,10 @@ typedef struct obj_s { \ (b).root = (a).root; \ \ - (b).offm = (a).offm; \ - (b).offn = (a).offn; \ - (b).m = (a).m; \ - (b).n = (a).n; \ + (b).off[0] = (a).off[0]; \ + (b).off[1] = (a).off[1]; \ + (b).dim[0] = (a).dim[0]; \ + (b).dim[1] = (a).dim[1]; \ (b).diag_off = (a).diag_off; \ \ (b).info = (a).info; \ @@ -669,8 +667,8 @@ typedef struct obj_s { \ (b).root = (a).root; \ \ - (b).offm = (a).offm; \ - (b).offn = (a).offn; \ + (b).off[0] = (a).off[0]; \ + (b).off[1] = (a).off[1]; \ /* Avoid copying m since it will be overwritten. */ \ /* Avoid copying n since it will be overwritten. */ \ (b).diag_off = (a).diag_off; \ @@ -727,6 +725,15 @@ typedef enum } subpart_t; +// -- Matrix dimension type -- + +typedef enum +{ + BLIS_M = 0, + BLIS_N = 1 +} mdim_t; + + // -- Machine parameter types -- typedef enum diff --git a/frame/include/blis.h b/frame/include/blis.h index 2b27908023..39f6ef42a4 100644 --- a/frame/include/blis.h +++ b/frame/include/blis.h @@ -113,6 +113,7 @@ extern "C" { #include "bli_pool.h" #include "bli_mem.h" #include "bli_part.h" +#include "bli_prune.h" #include "bli_query.h" #include "bli_blocksize.h" #include "bli_func.h" diff --git a/test/thread_ranges/Makefile b/test/thread_ranges/Makefile new file mode 100644 index 0000000000..b506f20abe --- /dev/null +++ b/test/thread_ranges/Makefile @@ -0,0 +1,203 @@ +#!/bin/bash +# +# BLIS +# An object-based framework for developing high-performance BLAS-like +# libraries. +# +# Copyright (C) 2014, The University of Texas at Austin +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# - Neither the name of The University of Texas at Austin nor the names +# of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# +# Makefile +# +# Field G. Van Zee +# +# Makefile for standalone BLIS test drivers. +# + +# +# --- Makefile PHONY target definitions ---------------------------------------- +# + +.PHONY: all \ + test-ranges \ + clean cleanx + + + +# +# --- Makefile initialization -------------------------------------------------- +# + +# Define the name of the configuration file. +CONFIG_MK_FILE := config.mk + +# Define the name of the file containing build and architecture-specific +# makefile definitions. +MAKE_DEFS_FILE := make_defs.mk + +# Locations of important files. +ROOT_PATH := ../.. +CONFIG_DIR := config + + + +# +# --- Include makefile configuration file -------------------------------------- +# + +# Construct the path to the makefile configuration file that was generated by +# the configure script. +CONFIG_MK_PATH := $(ROOT_PATH)/$(CONFIG_MK_FILE) + +# Include the configuration file. +-include $(CONFIG_MK_PATH) + +# Detect whether we actually got the configuration file. If we didn't, then +# it is likely that the user has not yet generated it (via configure). +ifeq ($(strip $(CONFIG_MK_INCLUDED)),yes) +CONFIG_MK_PRESENT := yes +else +CONFIG_MK_PRESENT := no +endif + +# Now we have access to CONFIG_NAME, which tells us which sub-directory of the +# config directory to use as our configuration. +CONFIG_PATH := $(ROOT_PATH)/$(CONFIG_DIR)/$(CONFIG_NAME) + + + +# +# --- Include makefile definitions file ---------------------------------------- +# + +# Construct the path to the makefile definitions file residing inside of +# the configuration sub-directory. +MAKE_DEFS_MK_PATH := $(CONFIG_PATH)/$(MAKE_DEFS_FILE) + +# Include the makefile definitions file. +-include $(MAKE_DEFS_MK_PATH) + +# Detect whether we actually got the make definitios file. If we didn't, then +# it is likely that the configuration is invalid (or incomplete). +ifeq ($(strip $(MAKE_DEFS_MK_INCLUDED)),yes) +MAKE_DEFS_MK_PRESENT := yes +else +MAKE_DEFS_MK_PRESENT := no +endif + + + +# +# --- BLAS and LAPACK implementations ------------------------------------------ +# + +# BLIS library and header path. This is simply wherever it was installed. +BLIS_LIB_PATH := $(INSTALL_PREFIX)/lib +BLIS_INC_PATH := $(INSTALL_PREFIX)/include/blis + +# BLIS library. +BLIS_LIB := $(BLIS_LIB_PATH)/libblis.a + + + +# +# --- General build definitions ------------------------------------------------ +# + +TEST_SRC_PATH := . +TEST_OBJ_PATH := . + +# Gather all local object files. +TEST_OBJS := $(patsubst $(TEST_SRC_PATH)/%.c, \ + $(TEST_OBJ_PATH)/%.o, \ + $(wildcard $(TEST_SRC_PATH)/*.c)) + +# Override CFLAGS from make_defs.mk here, if desired. +#CFLAGS := -g -O2 -march=native + +# Add installed and local header paths to CFLAGS +CFLAGS += -I$(BLIS_INC_PATH) -I$(TEST_SRC_PATH) #-I$(ACML_INC_PATH) + +LINKER := $(CC) +LDFLAGS := #-L/home/00146/field/gnu/gcc-4.8.2/lib64 +LDFLAGS += -lgfortran -lm -lpthread -fopenmp + + +# Datatype +DT_S := -DDT=BLIS_FLOAT +DT_D := -DDT=BLIS_DOUBLE +DT_C := -DDT=BLIS_SCOMPLEX +DT_Z := -DDT=BLIS_DCOMPLEX + +# Problem size specification +PDEF_MT := -DP_BEGIN=400 \ + -DP_END=8000 \ + -DP_INC=400 + + + +# +# --- Targets/rules ------------------------------------------------------------ +# + +all: test-ranges + +test-ranges: \ + test_ranges.x + + + +# --Object file rules -- + +$(TEST_OBJ_PATH)/%.o: $(TEST_SRC_PATH)/%.c + $(CC) $(CFLAGS) -c $< -o $@ + +# blis asm +test_%.o: test_%.c + $(CC) $(CFLAGS) $(PDEF_MT) $(DT_D) -c $< -o $@ + + +# -- Executable file rules -- + +# NOTE: For the BLAS test drivers, we place the BLAS libraries before BLIS +# on the link command line in case BLIS was configured with the BLAS +# compatibility layer. This prevents BLIS from inadvertently getting called +# for the BLAS routines we are trying to test with. + +test_ranges.x: test_ranges.o $(BLIS_LIB) + $(LINKER) $< $(BLIS_LIB) $(LDFLAGS) -o $@ + + +# -- Clean rules -- + +clean: cleanx + +cleanx: + - $(RM_F) *.o *.x + diff --git a/test/thread_ranges/test_ranges.c b/test/thread_ranges/test_ranges.c new file mode 100644 index 0000000000..b34a80cb08 --- /dev/null +++ b/test/thread_ranges/test_ranges.c @@ -0,0 +1,313 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include +#include "blis.h" + + +//#define PRINT + +int main( int argc, char** argv ) +{ + bli_init(); + +#if 0 + obj_t a, b, c; + obj_t aa, bb, cc; + dim_t m, n, k; + num_t dt; + uplo_t uploa, uplob, uploc; + + { + dt = BLIS_DOUBLE; + + m = 6; + k = 6; + n = 6; + + bli_obj_create( dt, m, k, 0, 0, &a ); + bli_obj_create( dt, k, n, 0, 0, &b ); + bli_obj_create( dt, m, n, 0, 0, &c ); + + uploa = BLIS_UPPER; + uploa = BLIS_LOWER; + bli_obj_set_struc( BLIS_TRIANGULAR, a ); + bli_obj_set_uplo( uploa, a ); + bli_obj_set_diag_offset( -2, a ); + + uplob = BLIS_UPPER; + uplob = BLIS_LOWER; + bli_obj_set_struc( BLIS_TRIANGULAR, b ); + bli_obj_set_uplo( uplob, b ); + bli_obj_set_diag_offset( -2, b ); + + uploc = BLIS_UPPER; + //uploc = BLIS_LOWER; + //uploc = BLIS_ZEROS; + //uploc = BLIS_DENSE; + bli_obj_set_struc( BLIS_HERMITIAN, c ); + //bli_obj_set_struc( BLIS_TRIANGULAR, c ); + bli_obj_set_uplo( uploc, c ); + bli_obj_set_diag_offset( 1, c ); + + bli_obj_alias_to( a, aa ); (void)aa; + bli_obj_alias_to( b, bb ); (void)bb; + bli_obj_alias_to( c, cc ); (void)cc; + + bli_randm( &a ); + bli_randm( &b ); + bli_randm( &c ); + //bli_mkherm( &a ); + //bli_mktrim( &a ); + + bli_prune_unref_mparts( &cc, BLIS_M, + &aa, BLIS_N ); + + bli_printm( "c orig", &c, "%4.1f", "" ); + bli_printm( "c alias", &cc, "%4.1f", "" ); + bli_printm( "a orig", &a, "%4.1f", "" ); + bli_printm( "a alias", &aa, "%4.1f", "" ); + //bli_obj_print( "a struct", &a ); + } +#endif + + dim_t p_begin, p_max, p_inc; + gint_t m_input, n_input; + char uploa_ch; + doff_t diagoffa; + dim_t bf; + dim_t n_way; + char part_dim_ch; + bool_t go_fwd; + char out_ch; + + obj_t a; + + thrinfo_t thrinfo; + dim_t m, n; + uplo_t uploa; + bool_t part_m_dim, part_n_dim; + bool_t go_bwd; + dim_t p; + num_t dt; + dim_t start, end; + + dim_t width; + siz_t area; + + gint_t t_begin, t_stop, t_inc; + dim_t t; + + if ( argc == 13 ) + { + sscanf( argv[1], "%lu", &p_begin ); + sscanf( argv[2], "%lu", &p_max ); + sscanf( argv[3], "%lu", &p_inc ); + sscanf( argv[4], "%ld", &m_input ); + sscanf( argv[5], "%ld", &n_input ); + sscanf( argv[6], "%c", &uploa_ch ); + sscanf( argv[7], "%ld", &diagoffa ); + sscanf( argv[8], "%lu", &bf ); + sscanf( argv[9], "%lu", &n_way ); + sscanf( argv[10], "%c", &part_dim_ch ); + sscanf( argv[11], "%lu", &go_fwd ); + sscanf( argv[12], "%c", &out_ch ); + } + else + { + printf( "\n" ); + printf( " %s\n", argv[0] ); + printf( "\n" ); + printf( " Simulate the dimension ranges assigned to threads when\n" ); + printf( " partitioning a matrix for parallelism in BLIS.\n" ); + printf( "\n" ); + printf( " Usage:\n" ); + printf( "\n" ); + printf( " %s p_beg p_max p_inc m n uplo doff bf n_way part_dim go_fwd out\n", argv[0] ); + printf( "\n" ); + printf( " p_beg: the first problem size p to test.\n" ); + printf( " p_max: the maximum problem size p to test.\n" ); + printf( " p_inc: the increase in problem size p between tests.\n" ); + printf( " m: the m dimension:\n" ); + printf( " n: the n dimension:\n" ); + printf( " if m,n = -1: bind m,n to problem size p.\n" ); + printf( " if m,n = 0: bind m,n to p_max.\n" ); + printf( " if m,n > 0: hold m,n = c constant for all p.\n" ); + printf( " uplo: the uplo field of the matrix being partitioned:\n" ); + printf( " 'l': lower-stored (BLIS_LOWER)\n" ); + printf( " 'u': upper-stored (BLIS_UPPER)\n" ); + printf( " 'd': densely-stored (BLIS_DENSE)\n" ); + printf( " doff: the diagonal offset of the matrix being partitioned.\n" ); + printf( " bf: the simulated blocking factor. all thread ranges must\n" ); + printf( " be a multiple of bf, except for the range that contains\n" ); + printf( " the edge case (if one exists). the blocking factor\n" ); + printf( " would typically correspond to a register blocksize.\n" ); + printf( " n_way: the number of ways of parallelism for which we are\n" ); + printf( " partitioning (i.e.: the number of threads, or thread\n" ); + printf( " groups).\n" ); + printf( " part_dim: the dimension to partition:\n" ); + printf( " 'm': partition the m dimension.\n" ); + printf( " 'n': partition the n dimension.\n" ); + printf( " go_fwd: the direction to partition:\n" ); + printf( " '1': forward, e.g. left-to-right (part_dim = 'm') or\n" ); + printf( " top-to-bottom (part_dim = 'n')\n" ); + printf( " '0': backward, e.g. right-to-left (part_dim = 'm') or\n" ); + printf( " bottom-to-top (part_dim = 'n')\n" ); + printf( " NOTE: reversing the direction does not change the\n" ); + printf( " subpartitions' widths, but it does change which end of\n" ); + printf( " the index range receives the edge case, if it exists.\n" ); + printf( " out: the type of output per thread-column:\n" ); + printf( " 'w': the width (and area) of the thread's subpartition\n" ); + printf( " 'r': the actual ranges of the thread's subpartition\n" ); + printf( " where the start and end points of each range are\n" ); + printf( " inclusive and exclusive, respectively.\n" ); + printf( "\n" ); + + exit(1); + } + + if ( m_input == 0 ) m_input = p_max; + if ( n_input == 0 ) n_input = p_max; + + if ( part_dim_ch == 'm' ) { part_m_dim = TRUE; part_n_dim = FALSE; } + else { part_m_dim = FALSE; part_n_dim = TRUE; } + + go_bwd = !go_fwd; + + if ( uploa_ch == 'l' ) uploa = BLIS_LOWER; + else if ( uploa_ch == 'u' ) uploa = BLIS_UPPER; + else uploa = BLIS_DENSE; + + if ( part_n_dim ) + { + if ( bli_is_upper( uploa ) ) { t_begin = n_way-1; t_stop = -1; t_inc = -1; } + else /* if lower or dense */ { t_begin = 0; t_stop = n_way; t_inc = 1; } + } + else // if ( part_m_dim ) + { + if ( bli_is_lower( uploa ) ) { t_begin = n_way-1; t_stop = -1; t_inc = -1; } + else /* if upper or dense */ { t_begin = 0; t_stop = n_way; t_inc = 1; } + } + + printf( "\n" ); + printf( " part: %3s doff: %3ld bf: %3ld output: %s\n", + ( part_n_dim ? ( go_fwd ? "l2r" : "r2l" ) + : ( go_fwd ? "t2b" : "b2t" ) ), + diagoffa, bf, + ( out_ch == 'w' ? "width(area)" : "ranges" ) ); + printf( " uplo: %3c nt: %3ld\n", uploa_ch, n_way ); + printf( "\n" ); + + printf( " " ); + for ( t = t_begin; t != t_stop; t += t_inc ) + { + if ( part_n_dim ) + { + if ( t == t_begin ) printf( "left... " ); + else if ( t == t_stop-t_inc ) printf( " ...right" ); + else printf( " " ); + } + else // if ( part_m_dim ) + { + if ( t == t_begin ) printf( "top... " ); + else if ( t == t_stop-t_inc ) printf( " ...bottom" ); + else printf( " " ); + } + } + printf( "\n" ); + + + printf( "%4c x %4c ", 'm', 'n' ); + for ( t = t_begin; t != t_stop; t += t_inc ) + { + printf( "%9s %lu ", "thread", t ); + } + printf( "\n" ); + printf( "-------------" ); + for ( t = t_begin; t != t_stop; t += t_inc ) + { + printf( "-------------" ); + } + printf( "\n" ); + + + for ( p = p_begin; p <= p_max; p += p_inc ) + { + if ( m_input < 0 ) m = ( dim_t )p; + else m = ( dim_t )m_input; + if ( n_input < 0 ) n = ( dim_t )p; + else n = ( dim_t )n_input; + + dt = BLIS_DOUBLE; + + bli_obj_create( dt, m, n, 0, 0, &a ); + + bli_obj_set_struc( BLIS_TRIANGULAR, a ); + bli_obj_set_uplo( uploa, a ); + bli_obj_set_diag_offset( diagoffa, a ); + + bli_randm( &a ); + + printf( "%4lu x %4lu ", m, n ); + + for ( t = t_begin; t != t_stop; t += t_inc ) + { + thrinfo.n_way = n_way; + thrinfo.work_id = t; + + if ( part_n_dim && go_fwd ) + area = bli_get_range_weighted_l2r( &thrinfo, &a, bf, &start, &end ); + else if ( part_n_dim && go_bwd ) + area = bli_get_range_weighted_r2l( &thrinfo, &a, bf, &start, &end ); + else if ( part_m_dim && go_fwd ) + area = bli_get_range_weighted_t2b( &thrinfo, &a, bf, &start, &end ); + else // ( part_m_dim && go_bwd ) + area = bli_get_range_weighted_b2t( &thrinfo, &a, bf, &start, &end ); + + width = end - start; + + if ( out_ch == 'w' ) printf( "%4lu(%6lu) ", width, area ); + else printf( "[%4lu,%4lu) ", start, end ); + } + + printf( "\n" ); + + bli_obj_free( &a ); + } + + bli_finalize(); + + return 0; +} + From bbebdb5793a8fd6aaf257012ab0272beaa04a0de Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Fri, 25 Sep 2015 14:47:27 -0500 Subject: [PATCH 20/38] Replaced README with README.md. Details: - Replaced the old (and short) README file with a much more comprehensive version written in github-flavored markdown. The new file is based on content taken from the old Google Code homepage. --- README | 49 ----------- README.md | 239 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 239 insertions(+), 49 deletions(-) delete mode 100644 README create mode 100644 README.md diff --git a/README b/README deleted file mode 100644 index 7ce5513dc5..0000000000 --- a/README +++ /dev/null @@ -1,49 +0,0 @@ - -BLIS framework -README ---- - -Thank you for deciding to try out the BLIS framework! - -BLIS is a portable framework for instantiating BLAS-like libraries. The -framework was designed to isolate essential kernels of computation that, -when optimized, immediately enable optimized implementations of most of -its commonly used and computationally intensive operations. - -BLIS has many features. For more detailed information about the project, -please check the BLIS homepage: - - http://code.google.com/p/blis/ - -You can keep in touch with developers and other users of the project by -joining one or more of the following mailing lists: - - o blis-announce - http://groups.google.com/group/blis-announce - Used only for announcements and other important messages regarding - BLIS. - - o blis-discuss - http://groups.google.com/group/blis-discuss - Please join and post to this mailing list if you have general questions - or feedback regarding BLIS. Application developers (end users) should - probably post here. - - o blis-devel - http://groups.google.com/group/blis-devel - Please join and post to this mailing list if you are a BLIS developer - (i.e., you are trying to use BLIS to create libraries, you want to - write kernels for the framework, or you are trying to modify or extend - the framework itself). - -Also, please read the LICENSE file for information on copying and -distributing this software. - -For a step-by-step guide on configuring, compiling, and installing BLIS, -please read the INSTALL file. Also, please check the BLIS website's wiki -page for other useful how-to guides. - -Thanks again for your interest in BLIS! - -Regards, - -Field G. Van Zee -field@cs.utexas.edu - diff --git a/README.md b/README.md new file mode 100644 index 0000000000..80b5e40f43 --- /dev/null +++ b/README.md @@ -0,0 +1,239 @@ +Introduction +------------ + +Thank you for deciding to try out the BLIS framework! + +BLIS is a portable software framework for instantiating high-performance +BLAS-like dense linear algebra libraries. The framework was designed to +isolate essential kernels of computation that, when optimized, immediately +enable optimized implementations of most of its commonly used and +computationally intensive operations. BLIS is written in [ISO +C99](http://en.wikipedia.org/wiki/C99) and available under a +[new/modified/3-clause BSD +license](http://opensource.org/licenses/BSD-3-Clause). While BLIS exports a +[new BLAS-like API](), it also includes a BLAS compatibility layer which gives +application developers access to BLIS implementations via traditional [BLAS +routine calls](http://www.netlib.org/lapack/lug/node145.html). + +For a thorough presentation of our framework, please read our recently accepted +journal article, ["BLIS: A Framework for Rapidly Instantiating BLAS +Functionality"](http://www.cs.utexas.edu/users/flame/pubs/blis1_toms_rev3.pdf). +For those who just want an executive summary, please see the next section. + +In a follow-up article, ["The BLIS Framework: Experiments in +Portability"](http://www.cs.utexas.edu/users/flame/pubs/blis2_toms_rev3.pdf), +we investigate using BLIS to instantiate level-3 BLAS implementations on a +variety of general-purpose, low-power, and multicore architectures. + +An IPDPS'14 conference paper titled ["Anatomy of High-Performance Many-Threaded +Matrix +Multiplication"](http://www.cs.utexas.edu/users/flame/pubs/blis3_ipdps14.pdf) +systematically explores the opportunities for parallelism within the five loops +that BLIS exposes in its matrix multiplication algorithm. + +It is our belief that BLIS offers substantial benefits in productivity when +compared to conventional approaches to developing BLAS libraries, as well as a +much-needed refinement of the BLAS interface, and thus constitutes a major +advance in dense linear algebra computation. While BLIS remains a +work-in-progress, we are excited to continue its development and further +cultivate its use within the community. + +Key Features +------------ + +BLIS offers several advantages over traditional BLAS libraries: + + * **Portability that doesn't impede high performance.** Portability was a top +priority of ours when creating BLIS. With zero additional effort on the part of +the developer, BLIS is configurable as a fully-functional reference +implementation. But more importantly, the framework identifies and isolates a +key set of computational kernels which, when optimized, immediately and +automatically optimize performance across virtually all level-2 and level-3 +BLIS operations. In this way, the framework acts as a productivity multiplier. +And since the optimized (non-portable) code is compartmentalized within these +few kernels, instantiating a high-performance BLIS library on a new +architecture is a relatively straightforward endeavour. + + * **Generalized matrix storage.** The BLIS framework exports interfaces that +allow one to specify both the row stride and column stride of a matrix. This +allows one to compute with matrices stored in column-major order, row-major +order, or by general stride. (This latter storage format is important for those +seeking to implement tensor contractions on multidimensional arrays.) +Furthermore, since BLIS tracks stride information for each matrix, operands of +different storage formats can be used within the same operation invocation. By +contrast, BLAS requires column-major storage. And while the CBLAS interface +supports row-major storage, it does not allow mixing storage formats. + + * **Full support for the complex domain.** BLIS operations are developed and +expressed in their most general form, which is typically in the complex domain. +These formulations then simplify elegantly down to the real domain, with +conjugations becoming no-ops. Unlike the BLAS, all input operands in BLIS that +allow transposition and conjugate-transposition also support conjugation +(without transposition), which obviates the need for thread-unsafe workarounds. +Also, where applicable, both complex symmetric and complex Hermitian forms are +supported. (BLAS omits some complex symmetric operations, such as `symv`, +`syr`, and `syr2`.) + + * **Advanced multithreading support.** BLIS allows multiple levels of +symmetric multithreading for nearly all level-3 operations. (Currently, users +may choose to obtain parallelism via either OpenMP or POSIX threads). This +means that matrices may be partitioned in multiple dimensions simultaneously to +attain scalable, high-performance parallelism on multicore and many-core +architectures. The key to this innovation is a thread-specific control tree +infrastructure which encodes information about the logical thread topology and +allows threads to query and communicate data amongst one another. BLIS also +employs so-called "quadratic partitioning" when computing dimension sub-ranges +for each thread, so that arbitrary diagonal offsets of structured matrices with +unreferend regions are taken into account to achieve proper load balance. + + * **Ease of use.** The BLIS framework, and the library of routines it +generates, are easy to use for end users, experts, and vendors alike. An +optional BLAS compatibility layer provides application developers with +backwards compatibility to existing BLAS-dependent codes. Or, one may adjust or +write their application to take advantage of new BLIS functionality (such as +generalized storage formats or additional complex operations) by calling BLIS +directly. BLIS's interfaces will feel familiar to many veterans of BLAS since +BLIS exports APIs with BLAS-like calling sequences. And experts will find +BLIS's internal object-based APIs a delight to use when customizing or writing +their own BLIS operations. (Objects are relatively lightweight `structs` and +passed by address, which helps tame function calling overhead.) + + * **Multilayered API and exposed kernels.** The BLIS framework exposes its +implementations in various layers, allowing expert developers to access exactly +the functionality desired. This layered interface includes that of the +lowest-level kernels, for those who wish to bypass the bulk of the framework. +Optimizations can occur at various levels, in part thanks to exposed packing +and unpacking facilities, which by default are highly parameterized and +flexible. + + * **Functionality that grows with the community's needs.** As its name +suggests, the BLIS framework is not a single library or static API, but rather +a nearly-complete template for instantiating high-performance BLAS-like +libraries. Furthermore, the framework is extensible, allowing developers to +leverage existing components to support new operations as they are identified. +If such operations require new kernels for optimal efficiency, the framework +and its APIs will be adjusted and extended accordingly. + + * **Code re-use.** Auto-generation approaches to achieving the aforementioned +goals tend to quickly lead to code bloat due to the multiple dimensions of +variation supported: operation (i.e. `gemm`, `herk`, `trmm`, etc.); parameter +case (i.e. side, [conjugate-]transposition, upper/lower storage, unit/non-unit +diagonal); datatype (i.e. single-/double-precision real/complex); matrix +storage (i.e. row-major, column-major, generalized); and algorithm (i.e. +partitioning path and kernel shape). These "brute force" approaches often +consider and optimize each operation or case combination in isolation, which is +less than ideal when the goal is to provide entire libraries. BLIS was designed +to be a complete framework for implementing basic linear algebra operations, +but supporting this vast amount of functionality in a manageable way required a +holistic design that employed careful abstractions, layering, and recycling of +generic (highly parameterized) codes, subject to the constraint that high +performance remain attainable. + + * **A foundation for mixed domain and/or mixed precision operations.** BLIS +was designed with the hope of one day allowing computation on real and complex +operands within the same operation. Similarly, we wanted to allow mixing +operands' floating-point precisions, or both domain and precision. +Unfortunately, this feature results in a significant amount of additional code, +mostly in level-2 and lower operations, thus, it is disabled by default. +However, mixing domains in level-3 operations is possible, in theory, with +almost no additional effort on the part of the library developer, and such +operations would remain capable of high performance. (Please note that this +functionality is still highly experimental and should be thought of as a +feature that will be more thoroughly implemented at some future date.) + +Discussion +---------- + +You can keep in touch with developers and other users of the project by joining one of the following mailing lists: + + * [blis-discuss](http://groups.google.com/group/blis-discuss): Please join and +post to this mailing list if you have general questions or feedback regarding +BLIS. Application developers (end users) should probably post here, unless they +have bug reports, in which case they should post to +[blis-devel](http://groups.google.com/group/blis-devel). + + * [blis-devel](http://groups.google.com/group/blis-devel): Please join and +post to this mailing list if you are a BLIS developer (i.e., you are trying to +use BLIS to create libraries, you want to write kernels for the framework, or +you are trying to modify or extend the framework itself). Also, if you would +like to submit a bug report, or discuss a possible bug, please use this list. +**Note:** Most of the interesting discussions happen here; don't be afraid to +join! + +Citations +--------- + +For those of you looking for the appropriate article to cite regarding BLIS, we +recommend citing our +[first ACM TOMS journal paper](http://www.cs.utexas.edu/users/flame/pubs/blis1_toms_rev3.pdf): + +``` +@article{BLIS1, + author = {Field G. {V}an~{Z}ee and Robert A. {v}an~{d}e~{G}eijn}, + title = {{BLIS}: A Framework for Rapidly Instantiating {BLAS} Functionality}, + journal = {ACM Transactions on Mathematical Software}, + volume = {41}, + number = {3}, + pages = {14:1--14:33}, + year = {2015}, + issue_date = {June 2015}, + url = {http://doi.acm.org/10.1145/2764454}, +} +``` + +You may also cite the +[second ACM TOMS journal paper](http://www.cs.utexas.edu/users/flame/pubs/blis2_toms_rev3.pdf): + +``` +@article{BLIS2, + author = {Field G. {V}an~{Z}ee and Tyler Smith and Francisco D. Igual and + Mikhail Smelyanskiy and Xianyi Zhang and Michael Kistler and Vernon Austel and + John Gunnels and Tze Meng Low and Bryan Marker and Lee Killough and + Robert A. {v}an~{d}e~{G}eijn}, + title = {The {BLIS} Framework: Experiments in Portability}, + journal = {ACM Transactions on Mathematical Software}, + year = 2015, + note = {Accepted}, +} +``` + +We also have a third paper, submitted to IPDPS 2014, on achieving +[multithreaded parallelism in BLIS](http://www.cs.utexas.edu/users/flame/pubs/blis3_ipdps14.pdf): + +``` +@inproceedings{BLIS3, + author = {Tyler M. Smith and Robert A. {v}an~{d}e~{G}eijn and Mikhail Smelyanskiy and + Jeff R. Hammond and Field G. {V}an~{Z}ee}, + title = {Anatomy of High-Performance Many-Threaded Matrix Multiplication}, + booktitle = {28th IEEE International Parallel \& Distributed Processing Symposium + (IPDPS 2014)}, + year = 2014, +} +``` + +A fourth paper, submitted to ACM TOMS, also exists, which proposes an +[analytical model](http://www.cs.utexas.edu/users/flame/pubs/TOMS-BLIS-Analytical.pdf) for determining blocksize parameters in BLIS: + +``` +@article{BLIS4, + author = {Tze Meng Low and Francisco D. Igual and Tyler M. Smith and Enrique S. Quintana-Ort\'{\i}}, + title = {Analytical Models for the {BLIS} Framework}, + journal = {ACM Transactions on Mathematical Software}, + year = 2015, + note = {Pending}, +} +``` + +Funding +------- + +This project and its associated research was partially sponsored by grants from +[Microsoft](http://www.microsoft.com/), [Intel](http://www.intel.com/), [Texas +Instruments](http://www.ti.com/), and [AMD](http://www.amd.com/), as well as +grants from the [National Science Foundation](http://www.nsf.gov/) (Awards +CCF-0917167 ACI-1148125/1340293, and CCF-1320112). + +_Any opinions, findings and conclusions or recommendations expressed in this +material are those of the author(s) and do not necessarily reflect the views of +the National Science Foundation (NSF)._ + From 55329906ecd7ce1ab910e4d30a29354a9172e7ea Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Sat, 26 Sep 2015 20:47:19 -0500 Subject: [PATCH 21/38] Minor edits to README.md, testsuite. Details: - Fixed typos in README.md. - Fixed column heading alignment for testsuite when matlab output is enabled. - Minor updates to test/3m4m/runme.sh and test/3m4m/Makefile. --- README.md | 4 ++-- test/3m4m/Makefile | 6 +++--- test/3m4m/runme.sh | 39 +++++++++++++++++++++++++++--------- testsuite/src/test_libblis.c | 8 +++++--- testsuite/src/test_libblis.h | 2 +- 5 files changed, 41 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 80b5e40f43..ab68af8165 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ automatically optimize performance across virtually all level-2 and level-3 BLIS operations. In this way, the framework acts as a productivity multiplier. And since the optimized (non-portable) code is compartmentalized within these few kernels, instantiating a high-performance BLIS library on a new -architecture is a relatively straightforward endeavour. +architecture is a relatively straightforward endeavor. * **Generalized matrix storage.** The BLIS framework exports interfaces that allow one to specify both the row stride and column stride of a matrix. This @@ -84,7 +84,7 @@ infrastructure which encodes information about the logical thread topology and allows threads to query and communicate data amongst one another. BLIS also employs so-called "quadratic partitioning" when computing dimension sub-ranges for each thread, so that arbitrary diagonal offsets of structured matrices with -unreferend regions are taken into account to achieve proper load balance. +unreferenced regions are taken into account to achieve proper load balance. * **Ease of use.** The BLIS framework, and the library of routines it generates, are easy to use for end users, experts, and vendors alike. An diff --git a/test/3m4m/Makefile b/test/3m4m/Makefile index e83ca4bc14..66eb8dec42 100644 --- a/test/3m4m/Makefile +++ b/test/3m4m/Makefile @@ -231,9 +231,9 @@ PDEF_ST := -DP_BEGIN=40 \ -DP_END=2000 \ -DP_INC=40 -PDEF_MT := -DP_BEGIN=400 \ - -DP_END=8000 \ - -DP_INC=400 +PDEF_MT := -DP_BEGIN=80 \ + -DP_END=4000 \ + -DP_INC=80 diff --git a/test/3m4m/runme.sh b/test/3m4m/runme.sh index 445e40bd0b..bb65a5db56 100755 --- a/test/3m4m/runme.sh +++ b/test/3m4m/runme.sh @@ -4,16 +4,24 @@ exec_root="test" out_root="output" +sys="blis" #sys="stampede" -sys="wahlberg" +#sys="wahlberg" # Bind threads to processors. #export OMP_PROC_BIND=true #export GOMP_CPU_AFFINITY="0 2 4 6 8 10 12 14 1 3 5 7 9 11 13 15" -export GOMP_CPU_AFFINITY="0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15" +#export GOMP_CPU_AFFINITY="0 1 2 3 4 5 6 7" +export GOMP_CPU_AFFINITY="0 1 2 3 4 5 6 7" +#export GOMP_CPU_AFFINITY="0 2 4 6 1 3 5 7" +#export GOMP_CPU_AFFINITY="0 4 1 5 2 6 3 7" # Modify LD_LIBRARY_PATH. -if [ ${sys} = "stampede" ]; then +if [ ${sys} = "blis" ]; then + + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH" + +elif [ ${sys} = "stampede" ]; then # A hack to use libiomp5 with gcc. export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/apps/intel/13/composer_xe_2013.2.146/compiler/lib/intel64" @@ -25,7 +33,15 @@ elif [ ${sys} = "wahlberg" ]; then fi # Threading scheme to use when multithreading -if [ ${sys} = "stampede" ]; then +if [ ${sys} = "blis" ]; then + + jc_nt=1 # 5th loop + ic_nt=4 # 3rd loop + jr_nt=1 # 2nd loop + ir_nt=1 # 1st loop + nt=4 + +elif [ ${sys} = "stampede" ]; then jc_nt=2 # 5th loop ic_nt=8 # 3rd loop @@ -43,8 +59,8 @@ elif [ ${sys} = "wahlberg" ]; then fi # Threadedness to test. -threads="st" # mt" -threads_r="st" # mt" +threads="st mt" # st mt" +threads_r="st mt" # mt" # Datatypes to test. dts="z c" @@ -56,7 +72,12 @@ test_ops="${l3_ops}" test_ops_r="${l3_ops}" # Complex domain implementations to test. -if [ ${sys} = "stampede" ]; then +if [ ${sys} = "blis" ]; then + + #test_impls="openblas mkl 3mhw_blis 3m3_blis 3m2_blis 3m1_blis 4mhw_blis 4m1b_blis 4m1a_blis" + test_impls="openblas 3mhw_blis 3m3_blis 3m2_blis 3m1_blis 4mhw_blis 4m1b_blis 4m1a_blis" + +elif [ ${sys} = "stampede" ]; then test_impls="openblas mkl asm_blis 3mhw_blis 3m3_blis 3m2_blis 3m1_blis 4mhw_blis 4m1b_blis 4m1a_blis" #test_impls="openblas mkl asm_blis" @@ -68,7 +89,8 @@ elif [ ${sys} = "wahlberg" ]; then fi # Real domain implementations to test. -test_impls_r="openblas acml asm_blis" +#test_impls_r="openblas mkl asm_blis" +test_impls_r="openblas asm_blis" # First perform real test cases. for th in ${threads_r}; do @@ -105,7 +127,6 @@ for th in ${threads_r}; do #export GOMP_CPU_AFFINITY="0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15" fi - # Construct the name of the test executable. exec_name="${exec_root}_${dt}${op}_${im}_${th}.x" diff --git a/testsuite/src/test_libblis.c b/testsuite/src/test_libblis.c index e6ae326480..55a98bd961 100644 --- a/testsuite/src/test_libblis.c +++ b/testsuite/src/test_libblis.c @@ -1444,7 +1444,7 @@ void libblis_test_op_driver( test_params_t* params, dt_char = params->datatype_char[dt]; // Build a commented column label string. - libblis_test_build_col_labels_string( op, label_str ); + libblis_test_build_col_labels_string( params, op, label_str ); // Output the column label string. libblis_test_fprintf( stdout, "%s\n", label_str ); @@ -1707,7 +1707,7 @@ void libblis_test_build_dims_string( test_op_t* op, // % dtoper_params_storage m n k gflops resid result -void libblis_test_build_col_labels_string( test_op_t* op, char* l_str ) +void libblis_test_build_col_labels_string( test_params_t* params, test_op_t* op, char* l_str ) { unsigned int n_spaces; char blank_str[64]; @@ -1727,7 +1727,9 @@ void libblis_test_build_col_labels_string( test_op_t* op, char* l_str ) "
_ " ); } - n_spaces = 1; + if ( params->output_matlab_format ) n_spaces = 11; + else n_spaces = 1; + fill_string_with_n_spaces( blank_str, n_spaces ); sprintf( &l_str[strlen(l_str)], "%s", blank_str ); diff --git a/testsuite/src/test_libblis.h b/testsuite/src/test_libblis.h index 6a747d4649..8bdbd37be3 100644 --- a/testsuite/src/test_libblis.h +++ b/testsuite/src/test_libblis.h @@ -362,7 +362,7 @@ void libblis_test_build_filename_string( char* prefix_str, char* op_str, char* funcname_str ); -void libblis_test_build_col_labels_string( test_op_t* op, char* l_str ); +void libblis_test_build_col_labels_string( test_params_t* params, test_op_t* op, char* l_str ); void fill_string_with_n_spaces( char* str, unsigned int n_spaces ); From e7e1f2f7b601b21b50e3cdad8972cb3fe11018d3 Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Fri, 2 Oct 2015 16:51:52 -0500 Subject: [PATCH 22/38] Minor updates to CREDITS, README files. --- CREDITS | 9 +++++---- README.md | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/CREDITS b/CREDITS index 79c97634e4..eea4cdee7e 100644 --- a/CREDITS +++ b/CREDITS @@ -12,9 +12,9 @@ but many others have contributed input and feedback, including: Vernon Austel (IBM, T.J. Watson Research Center) Jed Brown (Argonne National Laboratory) John Gunnels (IBM, T.J. Watson Research Center) - Jeff Hammond (Argonne National Laboratory) + Jeff Hammond (Intel) Francisco Igual (Universidad Complutense de Madrid) - Lee Killough (Appro International) + Lee Killough (Cray) Mike Kistler (IBM, Austin Research Laboratory) Bryan Marker (The University of Texas at Austin) Devin Matthews (The University of Texas at Austin) @@ -33,6 +33,7 @@ BLIS to new architectures as proofs-of-concept: Tyler Smith (The University of Texas at Austin) Zhang Xianyi (Chinese Academy of Sciences) -BLIS's development was partially funded by grants from Microsoft and the -National Science Foundation (NSF Awards CCF-0917167 and OCI-1148125). +BLIS's development was partially funded by grants from Microsoft, Intel, +Texas Instruments, AMD, and the National Science Foundation (NSF Awards +CCF-0917167, ACI-1148125/1340293, and CCF-1320112). diff --git a/README.md b/README.md index ab68af8165..4e5705b4ca 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ +![The BLIS cat is sleeping.](http://www.cs.utexas.edu/users/field/blis_cat.png) + Introduction ------------ -Thank you for deciding to try out the BLIS framework! - BLIS is a portable software framework for instantiating high-performance BLAS-like dense linear algebra libraries. The framework was designed to isolate essential kernels of computation that, when optimized, immediately From d17057446f5404824478e8a6cd08f242ab75544a Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Mon, 12 Oct 2015 11:39:49 -0500 Subject: [PATCH 23/38] Added "Getting Started" section to README.md. Details: - Added section to README.md file containing links to wikis with brief descriptions. --- README.md | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4e5705b4ca..676fa8084b 100644 --- a/README.md +++ b/README.md @@ -141,10 +141,46 @@ operations would remain capable of high performance. (Please note that this functionality is still highly experimental and should be thought of as a feature that will be more thoroughly implemented at some future date.) +Getting Started +--------------- + +If you just want to browse a quick-reference guide on user-level BLIS +interfaces, please read the [BLIS API quick +reference](https://github.com/flame/blis/wiki/BLISAPIQuickReference). +There you will find a brief description of each operation as well as some more +general information needed when developing an application with BLIS. + +Have a quick question? You may find the answer in our list of [frequently asked +questions](https://github.com/flame/blis/wiki/FAQ). + +Does BLIS contain kernels optimized for your favorite architecture? Please see +our [Hardware Support wiki](https://github.com/flame/blis/wiki/HardwareSupport) +for a full list of optimized kernels. + +We also provide wikis on the following topics, which will likely be of interest +to many users and developers: + * **[Build system](https://github.com/flame/blis/wiki/BuildSystem).** +This wiki provides step-by-step instructions for building a BLIS library. +(Reminder: While BLIS supports configure-time hardware detection for certain +architectures, you may need to manually specify a configuration to use.) + * **[Configuration](https://github.com/flame/blis/wiki/ConfigurationHowTo).** +This wiki describes how to create a BLIS "configuration", which captures all of +the details necessary to build BLIS for a specific hardware architecture. +Configurations specify things like blocksizes, kernel names, and various +optional configuration settings. + * **[Kernels](https://github.com/flame/blis/wiki/KernelsHowTo).** +This wiki describes each of the BLIS kernel operations in detail and should +provide developers with most of the information needed to get started with +writing and optimizing their own kernels. + * **[Test suite](https://github.com/flame/blis/wiki/Testsuite).** +This wiki contains detailed instructions on running the BLIS test suite, +located in the top-level directory testsuite. + Discussion ---------- -You can keep in touch with developers and other users of the project by joining one of the following mailing lists: +You can keep in touch with developers and other users of the project by joining +one of the following mailing lists: * [blis-discuss](http://groups.google.com/group/blis-discuss): Please join and post to this mailing list if you have general questions or feedback regarding From 276da366187460a4c8e6e0910e79cb39ce780bfe Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Mon, 12 Oct 2015 11:43:03 -0500 Subject: [PATCH 24/38] Minor formatting change to README.md. --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 676fa8084b..c6840edef0 100644 --- a/README.md +++ b/README.md @@ -159,20 +159,20 @@ for a full list of optimized kernels. We also provide wikis on the following topics, which will likely be of interest to many users and developers: - * **[Build system](https://github.com/flame/blis/wiki/BuildSystem).** + * [Build system](https://github.com/flame/blis/wiki/BuildSystem). This wiki provides step-by-step instructions for building a BLIS library. (Reminder: While BLIS supports configure-time hardware detection for certain architectures, you may need to manually specify a configuration to use.) - * **[Configuration](https://github.com/flame/blis/wiki/ConfigurationHowTo).** + * [Configuration](https://github.com/flame/blis/wiki/ConfigurationHowTo). This wiki describes how to create a BLIS "configuration", which captures all of the details necessary to build BLIS for a specific hardware architecture. Configurations specify things like blocksizes, kernel names, and various optional configuration settings. - * **[Kernels](https://github.com/flame/blis/wiki/KernelsHowTo).** + * [Kernels](https://github.com/flame/blis/wiki/KernelsHowTo). This wiki describes each of the BLIS kernel operations in detail and should provide developers with most of the information needed to get started with writing and optimizing their own kernels. - * **[Test suite](https://github.com/flame/blis/wiki/Testsuite).** + * [Test suite](https://github.com/flame/blis/wiki/Testsuite). This wiki contains detailed instructions on running the BLIS test suite, located in the top-level directory testsuite. From 77ddb0b1d31ada111dadf392766ba6d9210ed9fb Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Tue, 13 Oct 2015 12:53:06 -0500 Subject: [PATCH 25/38] Removed flop-counting mechanism. Details: - Removed the optional flop-counting feature introduced in commit 7574c994. --- frame/3/gemm/bli_gemm_front.c | 7 ----- frame/3/hemm/bli_hemm_front.c | 7 ----- frame/3/her2k/bli_her2k_front.c | 7 ----- frame/3/herk/bli_herk_front.c | 7 ----- frame/3/symm/bli_symm_front.c | 7 ----- frame/3/syr2k/bli_syr2k_front.c | 7 ----- frame/3/syrk/bli_syrk_front.c | 7 ----- frame/3/trmm/bli_trmm_front.c | 7 ----- frame/3/trmm3/bli_trmm3_front.c | 7 ----- frame/3/trsm/bli_trsm_front.c | 7 ----- frame/base/bli_flops.c | 53 --------------------------------- frame/base/bli_flops.h | 38 ----------------------- frame/include/blis.h | 1 - 13 files changed, 162 deletions(-) delete mode 100644 frame/base/bli_flops.c delete mode 100644 frame/base/bli_flops.h diff --git a/frame/3/gemm/bli_gemm_front.c b/frame/3/gemm/bli_gemm_front.c index 6aa49b706c..78220aff34 100644 --- a/frame/3/gemm/bli_gemm_front.c +++ b/frame/3/gemm/bli_gemm_front.c @@ -97,12 +97,5 @@ void bli_gemm_front( obj_t* alpha, bli_gemm_thrinfo_free_paths( infos, n_threads ); -#ifdef BLIS_ENABLE_FLOP_COUNT - // Increment the global flop counter. - bli_flop_count_inc( 2.0 * bli_obj_length( *c ) - * bli_obj_width( *c ) - * bli_obj_width_after_trans( a_local ) - * ( bli_obj_is_complex( *c ) ? 4.0 : 1.0 ) ); -#endif } diff --git a/frame/3/hemm/bli_hemm_front.c b/frame/3/hemm/bli_hemm_front.c index 4a1c2c36e8..4d941967d6 100644 --- a/frame/3/hemm/bli_hemm_front.c +++ b/frame/3/hemm/bli_hemm_front.c @@ -104,12 +104,5 @@ void bli_hemm_front( side_t side, bli_gemm_thrinfo_free_paths( infos, n_threads ); -#ifdef BLIS_ENABLE_FLOP_COUNT - // Increment the global flop counter. - bli_flop_count_inc( 2.0 * bli_obj_length( *c ) - * bli_obj_width( *c ) - * bli_obj_width_after_trans( a_local ) - * ( bli_obj_is_complex( *c ) ? 4.0 : 1.0 ) ); -#endif } diff --git a/frame/3/her2k/bli_her2k_front.c b/frame/3/her2k/bli_her2k_front.c index 0bc4e91608..25bc59eb7a 100644 --- a/frame/3/her2k/bli_her2k_front.c +++ b/frame/3/her2k/bli_her2k_front.c @@ -156,12 +156,5 @@ void bli_her2k_front( obj_t* alpha, // to zero before returning. bli_setid( &BLIS_ZERO, &c_local ); -#ifdef BLIS_ENABLE_FLOP_COUNT - // Increment the global flop counter. - bli_flop_count_inc( 2.0 * bli_obj_length( *c ) - * bli_obj_width( *c ) - * bli_obj_width_after_trans( a_local ) - * ( bli_obj_is_complex( *c ) ? 4.0 : 1.0 ) ); -#endif } diff --git a/frame/3/herk/bli_herk_front.c b/frame/3/herk/bli_herk_front.c index 3cd244ac60..1129f0690e 100644 --- a/frame/3/herk/bli_herk_front.c +++ b/frame/3/herk/bli_herk_front.c @@ -110,12 +110,5 @@ void bli_herk_front( obj_t* alpha, // to zero before returning. bli_setid( &BLIS_ZERO, &c_local ); -#ifdef BLIS_ENABLE_FLOP_COUNT - // Increment the global flop counter. - bli_flop_count_inc( 1.0 * bli_obj_length( *c ) - * bli_obj_width( *c ) - * bli_obj_width_after_trans( a_local ) - * ( bli_obj_is_complex( *c ) ? 4.0 : 1.0 ) ); -#endif } diff --git a/frame/3/symm/bli_symm_front.c b/frame/3/symm/bli_symm_front.c index d223708ec1..5583d7e0f3 100644 --- a/frame/3/symm/bli_symm_front.c +++ b/frame/3/symm/bli_symm_front.c @@ -103,12 +103,5 @@ void bli_symm_front( side_t side, bli_gemm_thrinfo_free_paths( infos, n_threads ); -#ifdef BLIS_ENABLE_FLOP_COUNT - // Increment the global flop counter. - bli_flop_count_inc( 2.0 * bli_obj_length( *c ) - * bli_obj_width( *c ) - * bli_obj_width_after_trans( a_local ) - * ( bli_obj_is_complex( *c ) ? 4.0 : 1.0 ) ); -#endif } diff --git a/frame/3/syr2k/bli_syr2k_front.c b/frame/3/syr2k/bli_syr2k_front.c index e3475fc44d..ca7a2a8fe1 100644 --- a/frame/3/syr2k/bli_syr2k_front.c +++ b/frame/3/syr2k/bli_syr2k_front.c @@ -127,12 +127,5 @@ void bli_syr2k_front( obj_t* alpha, bli_herk_thrinfo_free_paths( infos, n_threads ); #endif -#ifdef BLIS_ENABLE_FLOP_COUNT - // Increment the global flop counter. - bli_flop_count_inc( 2.0 * bli_obj_length( *c ) - * bli_obj_width( *c ) - * bli_obj_width_after_trans( a_local ) - * ( bli_obj_is_complex( *c ) ? 4.0 : 1.0 ) ); -#endif } diff --git a/frame/3/syrk/bli_syrk_front.c b/frame/3/syrk/bli_syrk_front.c index 53ba3485dc..46529cde76 100644 --- a/frame/3/syrk/bli_syrk_front.c +++ b/frame/3/syrk/bli_syrk_front.c @@ -96,12 +96,5 @@ void bli_syrk_front( obj_t* alpha, bli_herk_thrinfo_free_paths( infos, n_threads ); -#ifdef BLIS_ENABLE_FLOP_COUNT - // Increment the global flop counter. - bli_flop_count_inc( 1.0 * bli_obj_length( *c ) - * bli_obj_width( *c ) - * bli_obj_width_after_trans( a_local ) - * ( bli_obj_is_complex( *c ) ? 4.0 : 1.0 ) ); -#endif } diff --git a/frame/3/trmm/bli_trmm_front.c b/frame/3/trmm/bli_trmm_front.c index 2b69df9abf..67594a1c23 100644 --- a/frame/3/trmm/bli_trmm_front.c +++ b/frame/3/trmm/bli_trmm_front.c @@ -144,12 +144,5 @@ void bli_trmm_front( side_t side, bli_trmm_thrinfo_free_paths( infos, n_threads ); -#ifdef BLIS_ENABLE_FLOP_COUNT - // Increment the global flop counter. - bli_flop_count_inc( 1.0 * bli_obj_length( *c ) - * bli_obj_width( *c ) - * bli_obj_width_after_trans( a_local ) - * ( bli_obj_is_complex( *c ) ? 4.0 : 1.0 ) ); -#endif } diff --git a/frame/3/trmm3/bli_trmm3_front.c b/frame/3/trmm3/bli_trmm3_front.c index 9cb3f9091d..0a90f68306 100644 --- a/frame/3/trmm3/bli_trmm3_front.c +++ b/frame/3/trmm3/bli_trmm3_front.c @@ -147,12 +147,5 @@ void bli_trmm3_front( side_t side, bli_trmm_thrinfo_free_paths( infos, n_threads ); -#ifdef BLIS_ENABLE_FLOP_COUNT - // Increment the global flop counter. - bli_flop_count_inc( 1.0 * bli_obj_length( *c ) - * bli_obj_width( *c ) - * bli_obj_width_after_trans( a_local ) - * ( bli_obj_is_complex( *c ) ? 4.0 : 1.0 ) ); -#endif } diff --git a/frame/3/trsm/bli_trsm_front.c b/frame/3/trsm/bli_trsm_front.c index b26ead61f0..554dc36c62 100644 --- a/frame/3/trsm/bli_trsm_front.c +++ b/frame/3/trsm/bli_trsm_front.c @@ -130,12 +130,5 @@ void bli_trsm_front( side_t side, bli_trsm_thrinfo_free_paths( infos, n_threads ); -#ifdef BLIS_ENABLE_FLOP_COUNT - // Increment the global flop counter. - bli_flop_count_inc( 1.0 * bli_obj_length( *c ) - * bli_obj_width( *c ) - * bli_obj_width_after_trans( a_local ) - * ( bli_obj_is_complex( *c ) ? 4.0 : 1.0 ) ); -#endif } diff --git a/frame/base/bli_flops.c b/frame/base/bli_flops.c deleted file mode 100644 index 19f37ec3ad..0000000000 --- a/frame/base/bli_flops.c +++ /dev/null @@ -1,53 +0,0 @@ -/* - - BLIS - An object-based framework for developing high-performance BLAS-like - libraries. - - Copyright (C) 2014, The University of Texas at Austin - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - Neither the name of The University of Texas at Austin nor the names - of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -#include "blis.h" - -static double bli_flops_executed = 0.0; - -void bli_flop_count_inc( double new_flops ) -{ - bli_flops_executed += new_flops; -} - -double bli_flop_count( void ) -{ - return bli_flops_executed; -} - -void bli_flop_count_reset( void ) -{ - bli_flops_executed = 0.0; -} - diff --git a/frame/base/bli_flops.h b/frame/base/bli_flops.h deleted file mode 100644 index c976fa1cb3..0000000000 --- a/frame/base/bli_flops.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - - BLIS - An object-based framework for developing high-performance BLAS-like - libraries. - - Copyright (C) 2014, The University of Texas at Austin - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - Neither the name of The University of Texas at Austin nor the names - of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -void bli_flop_count_inc( double new_flops ); -double bli_flop_count( void ); -void bli_flop_count_reset( void ); - diff --git a/frame/include/blis.h b/frame/include/blis.h index 39f6ef42a4..057c734bfc 100644 --- a/frame/include/blis.h +++ b/frame/include/blis.h @@ -125,7 +125,6 @@ extern "C" { #include "bli_machval.h" #include "bli_getopt.h" #include "bli_opid.h" -#include "bli_flops.h" #include "bli_cntl.h" #include "bli_info.h" From 4f88c29f9e634cbb6fb22d8c88931f0ec78ad7db Mon Sep 17 00:00:00 2001 From: Zhang Xianyi Date: Wed, 14 Oct 2015 12:57:50 -0500 Subject: [PATCH 26/38] Detect Intel Broadwell (using Haswell config). --- build/auto-detect/cpuid_x86.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/build/auto-detect/cpuid_x86.c b/build/auto-detect/cpuid_x86.c index 6bbaf9f4e4..9a703c0855 100644 --- a/build/auto-detect/cpuid_x86.c +++ b/build/auto-detect/cpuid_x86.c @@ -172,6 +172,7 @@ int cpu_detect() case 12: case 15: //Haswell + case 13: //Broadwell if(support_avx()) { return CPUNAME_HASWELL; }else{ @@ -185,6 +186,9 @@ int cpu_detect() case 5: case 6: //Haswell + case 7: + case 15: + //Broadwell if(support_avx()) { return CPUNAME_HASWELL; }else{ @@ -192,6 +196,17 @@ int cpu_detect() } } break; + case 5: + switch (model) { + case 6: + //Broadwell + if(support_avx()) { + return CPUNAME_HASWELL; + }else{ + return CPUNAME_REFERENCE; //OS doesn't support AVX + } + } + break; } break; } From b489152e112644ec3b6d19e687231a9607f7694f Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Wed, 21 Oct 2015 14:53:17 -0500 Subject: [PATCH 27/38] Use vzeroall in haswell micro-kernels. --- kernels/x86_64/avx2/3/bli_gemm_asm_d12x4.c | 28 ++++------------------ kernels/x86_64/avx2/3/bli_gemm_asm_d8x6.c | 28 ++++------------------ 2 files changed, 8 insertions(+), 48 deletions(-) diff --git a/kernels/x86_64/avx2/3/bli_gemm_asm_d12x4.c b/kernels/x86_64/avx2/3/bli_gemm_asm_d12x4.c index 09fab6b473..2f9ec6542f 100644 --- a/kernels/x86_64/avx2/3/bli_gemm_asm_d12x4.c +++ b/kernels/x86_64/avx2/3/bli_gemm_asm_d12x4.c @@ -84,6 +84,8 @@ void bli_sgemm_asm_24x4( __asm__ volatile ( " \n\t" + "vzeroall \n\t" // zero all xmm/ymm registers. + " \n\t" " \n\t" "movq %2, %%rax \n\t" // load address of a. "movq %3, %%rbx \n\t" // load address of b. @@ -104,18 +106,6 @@ void bli_sgemm_asm_24x4( "prefetcht0 7 * 4(%%rcx,%%rdi,2) \n\t" // prefetch c + 2*cs_c "prefetcht0 7 * 4(%%rcx,%%r13) \n\t" // prefetch c + 3*cs_c " \n\t" - "vxorps %%ymm4, %%ymm4, %%ymm4 \n\t" - "vxorps %%ymm5, %%ymm5, %%ymm5 \n\t" - "vxorps %%ymm6, %%ymm6, %%ymm6 \n\t" - "vxorps %%ymm7, %%ymm7, %%ymm7 \n\t" - "vxorps %%ymm8, %%ymm8, %%ymm8 \n\t" - "vxorps %%ymm9, %%ymm9, %%ymm9 \n\t" - "vxorps %%ymm10, %%ymm10, %%ymm10 \n\t" - "vxorps %%ymm11, %%ymm11, %%ymm11 \n\t" - "vxorps %%ymm12, %%ymm12, %%ymm12 \n\t" - "vxorps %%ymm13, %%ymm13, %%ymm13 \n\t" - "vxorps %%ymm14, %%ymm14, %%ymm14 \n\t" - "vxorps %%ymm15, %%ymm15, %%ymm15 \n\t" " \n\t" " \n\t" " \n\t" @@ -711,6 +701,8 @@ void bli_dgemm_asm_12x4( __asm__ volatile ( " \n\t" + "vzeroall \n\t" // zero all xmm/ymm registers. + " \n\t" " \n\t" "movq %2, %%rax \n\t" // load address of a. "movq %3, %%rbx \n\t" // load address of b. @@ -731,18 +723,6 @@ void bli_dgemm_asm_12x4( "prefetcht0 7 * 8(%%rcx,%%rdi,2) \n\t" // prefetch c + 2*cs_c "prefetcht0 7 * 8(%%rcx,%%r13) \n\t" // prefetch c + 3*cs_c " \n\t" - "vxorpd %%ymm4, %%ymm4, %%ymm4 \n\t" - "vxorpd %%ymm5, %%ymm5, %%ymm5 \n\t" - "vxorpd %%ymm6, %%ymm6, %%ymm6 \n\t" - "vxorpd %%ymm7, %%ymm7, %%ymm7 \n\t" - "vxorpd %%ymm8, %%ymm8, %%ymm8 \n\t" - "vxorpd %%ymm9, %%ymm9, %%ymm9 \n\t" - "vxorpd %%ymm10, %%ymm10, %%ymm10 \n\t" - "vxorpd %%ymm11, %%ymm11, %%ymm11 \n\t" - "vxorpd %%ymm12, %%ymm12, %%ymm12 \n\t" - "vxorpd %%ymm13, %%ymm13, %%ymm13 \n\t" - "vxorpd %%ymm14, %%ymm14, %%ymm14 \n\t" - "vxorpd %%ymm15, %%ymm15, %%ymm15 \n\t" " \n\t" " \n\t" " \n\t" diff --git a/kernels/x86_64/avx2/3/bli_gemm_asm_d8x6.c b/kernels/x86_64/avx2/3/bli_gemm_asm_d8x6.c index 9c11950fc7..47cc7ad68b 100644 --- a/kernels/x86_64/avx2/3/bli_gemm_asm_d8x6.c +++ b/kernels/x86_64/avx2/3/bli_gemm_asm_d8x6.c @@ -84,6 +84,8 @@ void bli_sgemm_asm_16x6( __asm__ volatile ( " \n\t" + "vzeroall \n\t" // zero all xmm/ymm registers. + " \n\t" " \n\t" "movq %2, %%rax \n\t" // load address of a. "movq %3, %%rbx \n\t" // load address of b. @@ -107,18 +109,6 @@ void bli_sgemm_asm_16x6( "prefetcht0 7 * 8(%%rdx,%%rdi) \n\t" // prefetch c + 4*cs_c "prefetcht0 7 * 8(%%rdx,%%rdi,2) \n\t" // prefetch c + 5*cs_c " \n\t" - "vxorps %%ymm4, %%ymm4, %%ymm4 \n\t" - "vxorps %%ymm5, %%ymm5, %%ymm5 \n\t" - "vxorps %%ymm6, %%ymm6, %%ymm6 \n\t" - "vxorps %%ymm7, %%ymm7, %%ymm7 \n\t" - "vxorps %%ymm8, %%ymm8, %%ymm8 \n\t" - "vxorps %%ymm9, %%ymm9, %%ymm9 \n\t" - "vxorps %%ymm10, %%ymm10, %%ymm10 \n\t" - "vxorps %%ymm11, %%ymm11, %%ymm11 \n\t" - "vxorps %%ymm12, %%ymm12, %%ymm12 \n\t" - "vxorps %%ymm13, %%ymm13, %%ymm13 \n\t" - "vxorps %%ymm14, %%ymm14, %%ymm14 \n\t" - "vxorps %%ymm15, %%ymm15, %%ymm15 \n\t" " \n\t" " \n\t" " \n\t" @@ -705,6 +695,8 @@ void bli_dgemm_asm_8x6( __asm__ volatile ( " \n\t" + "vzeroall \n\t" // zero all xmm/ymm registers. + " \n\t" " \n\t" "movq %2, %%rax \n\t" // load address of a. "movq %3, %%rbx \n\t" // load address of b. @@ -728,18 +720,6 @@ void bli_dgemm_asm_8x6( "prefetcht0 7 * 8(%%rdx,%%rdi) \n\t" // prefetch c + 4*cs_c "prefetcht0 7 * 8(%%rdx,%%rdi,2) \n\t" // prefetch c + 5*cs_c " \n\t" - "vxorpd %%ymm4, %%ymm4, %%ymm4 \n\t" - "vxorpd %%ymm5, %%ymm5, %%ymm5 \n\t" - "vxorpd %%ymm6, %%ymm6, %%ymm6 \n\t" - "vxorpd %%ymm7, %%ymm7, %%ymm7 \n\t" - "vxorpd %%ymm8, %%ymm8, %%ymm8 \n\t" - "vxorpd %%ymm9, %%ymm9, %%ymm9 \n\t" - "vxorpd %%ymm10, %%ymm10, %%ymm10 \n\t" - "vxorpd %%ymm11, %%ymm11, %%ymm11 \n\t" - "vxorpd %%ymm12, %%ymm12, %%ymm12 \n\t" - "vxorpd %%ymm13, %%ymm13, %%ymm13 \n\t" - "vxorpd %%ymm14, %%ymm14, %%ymm14 \n\t" - "vxorpd %%ymm15, %%ymm15, %%ymm15 \n\t" " \n\t" " \n\t" " \n\t" From a0a7b85ac3e157af53cff8db0e008f4a3f90372c Mon Sep 17 00:00:00 2001 From: Francisco Igual Date: Tue, 27 Oct 2015 08:59:15 +0000 Subject: [PATCH 28/38] Fixed incomplete code in the double precision ARMv8 microkernel. --- config/armv8a/bli_kernel.h | 10 +- config/armv8a/make_defs.mk | 18 +- kernels/armv8a/neon/3/bli_gemm_opt_4x4.c | 294 +++++++++++++++++++---- 3 files changed, 260 insertions(+), 62 deletions(-) diff --git a/config/armv8a/bli_kernel.h b/config/armv8a/bli_kernel.h index 2e27051226..3bd7da7222 100644 --- a/config/armv8a/bli_kernel.h +++ b/config/armv8a/bli_kernel.h @@ -51,12 +51,12 @@ // (b) MR (for zero-padding purposes when MR and NR are "swapped") // -#define BLIS_DEFAULT_MC_S 416 // 1280 //160 // 160 // 160 //2048 //336 -#define BLIS_DEFAULT_KC_S 704 //1280 //672 //528 // 856 //2048 //528 +#define BLIS_DEFAULT_MC_S 336 +#define BLIS_DEFAULT_KC_S 336 #define BLIS_DEFAULT_NC_S 4096 -#define BLIS_DEFAULT_MC_D 80 //176 -#define BLIS_DEFAULT_KC_D 336 //368 +#define BLIS_DEFAULT_MC_D 160 +#define BLIS_DEFAULT_KC_D 304 #define BLIS_DEFAULT_NC_D 4096 #define BLIS_DEFAULT_MC_C 64 @@ -132,8 +132,6 @@ //#define BLIS_PACKDIM_MR_Z (BLIS_DEFAULT_MR_Z + ...) //#define BLIS_PACKDIM_NR_Z (BLIS_DEFAULT_NR_Z + ...) - - // -- LEVEL-2 KERNEL CONSTANTS ------------------------------------------------- diff --git a/config/armv8a/make_defs.mk b/config/armv8a/make_defs.mk index e81dc50260..63c03c6a09 100644 --- a/config/armv8a/make_defs.mk +++ b/config/armv8a/make_defs.mk @@ -76,20 +76,18 @@ GIT_LOG := $(GIT) log --decorate # # --- Determine the C compiler and related flags --- -##CC := gcc -CC := aarch64-linux-gnu-gcc -#CC := arm-linux-gnueabihf-gcc-4.9.2 +CC := gcc + # Enable IEEE Standard 1003.1-2004 (POSIX.1d). # NOTE: This is needed to enable posix_memalign(). -CPPROCFLAGS := -D_POSIX_C_SOURCE=200112L -#CMISCFLAGS := -std=c99 -mtune=cortex-a57 -mfpu=neon-fp-armv8 -march=armv8-a #-mfloat-abi=hard -mfpu=neon -CMISCFLAGS := -std=c99 -march=armv8-a+fp+simd -ftree-vectorize -O3 -mcpu=cortex-a57 -mtune=cortex-a57 #-mtune=cortex-a57 -march=armv8-a -mfloat-abi=hard -mfpu=neon +CPPROCFLAGS := -D_GNU_SOURCE +CMISCFLAGS := -std=c99 -march=armv8-a+fp+simd -ftree-vectorize -O3 -fopenmp -mcpu=cortex-a57.cortex-a53 -mtune=cortex-a57.cortex-a53 CPICFLAGS := -fPIC -CDBGFLAGS := -g +CDBGFLAGS := -g #-g3 -gdwarf-2 CWARNFLAGS := -Wall -COPTFLAGS := -march=armv8-a+fp+simd -ftree-vectorize -O3 -mcpu=cortex-a57 -mtune=cortex-a57 #-march=armv8-a -O2 -mtune=cortex-a57 -mfpu=neon-fp-armv8 #-mfpu=neon -O2 +COPTFLAGS := -march=armv8-a+fp+simd -ftree-vectorize -O3 -mcpu=cortex-a57.cortex-a53 -mtune=cortex-a57.cortex-a53 CKOPTFLAGS := $(COPTFLAGS) -CVECFLAGS := -march=armv8-a+fp+simd -ftree-vectorize -O3 -mcpu=cortex-a57 -mtune=cortex-a57 #-march=armv8-a -O2 -mtune=cortex-a57 -mfpu=neon-fp-armv8 +CVECFLAGS := #-march=armv8-a+fp+simd -ftree-vectorize -O3 -mcpu=cortex-a57.cortex-a53 -mtune=cortex-a57.cortex-a53 #-march=armv8-a -O2 -mtune=cortex-a57 -mfpu=neon-fp-armv8 # Aggregate all of the flags into multiple groups: one for standard # compilation, and one for each of the supported "special" compilation @@ -105,7 +103,7 @@ ARFLAGS := cru # --- Determine the linker and related flags --- LINKER := $(CC) SOFLAGS := -shared -LDFLAGS := -lm +LDFLAGS := -lm -fopenmp diff --git a/kernels/armv8a/neon/3/bli_gemm_opt_4x4.c b/kernels/armv8a/neon/3/bli_gemm_opt_4x4.c index ba98d8dd32..2a54fe8259 100644 --- a/kernels/armv8a/neon/3/bli_gemm_opt_4x4.c +++ b/kernels/armv8a/neon/3/bli_gemm_opt_4x4.c @@ -1,4 +1,4 @@ -/* + /* BLIS An object-based framework for developing high-performance BLAS-like @@ -36,12 +36,7 @@ #include "blis.h" /* - o 4x4 Single precision micro-kernel fully functional. - o Runnable on ARMv8, compiled with aarch64 GCC. - o Use it together with the armv8 BLIS configuration. o Tested on Juno board. Around 7.3 GFLOPS @ 1.1 GHz. - - December 2014. */ void bli_sgemm_opt_4x4( dim_t k, @@ -55,15 +50,15 @@ void bli_sgemm_opt_4x4( { void* a_next = bli_auxinfo_next_a( data ); void* b_next = bli_auxinfo_next_b( data ); - + dim_t k_iter = k / 4; - dim_t k_left = k % 4; + dim_t k_left = k % 4; -__asm__ volatile +__asm__ volatile ( " \n\t" " \n\t" -" ldr x0,%[aaddr] \n\t" // Load address of A. +" ldr x0,%[aaddr] \n\t" // Load address of A. " ldr x1,%[baddr] \n\t" // Load address of B. " ldr x2,%[caddr] \n\t" // Load address of C. " \n\t" @@ -433,12 +428,8 @@ __asm__ volatile /* - o 4x4 Double precision micro-kernel NOT fully functional yet. - o Runnable on ARMv8, compiled with aarch64 GCC. - o Use it together with the armv8 BLIS configuration. - o Tested on Juno board. Around 3 GFLOPS @ 1.1 GHz. - - December 2014. + * Tested on Juno board. Around 5.6 GFLOPS, 2 A57 cores @ 1.1 GHz. + * Tested on Juno board. Around 4 GFLOPS, 4 A53 cores @ 850 MHz. */ void bli_dgemm_opt_4x4( dim_t k, @@ -458,8 +449,8 @@ void bli_dgemm_opt_4x4( __asm__ volatile ( -" \n\t" -" ldr x0,%[aaddr] \n\t" // Load address of A +" \n\t" +" ldr x0,%[aaddr] \n\t" // Load address of A " ldr x1,%[baddr] \n\t" // Load address of B " ldr x2,%[caddr] \n\t" // Load address of C " \n\t" @@ -476,9 +467,16 @@ __asm__ volatile " \n\t" " ldr x9,%[cs_c] \n\t" // Load cs_c " lsl x10,x9,#3 \n\t" // cs_c * sizeof(double) -" \n\t" -" ldp q0,q1,[x0],32 \n\t" // Load a into quad -" ldp q4,q5,[x1],32 \n\t" // Load b into quad +" lsl x11,x9,#4 \n\t" // 2 * cs_c * sizeof(double) -- AUX. +" lsl x12,x9,#5 \n\t" // 3 * cs_c * sizeof(double) -- AUX. +" \n\t" +" ldr x13,%[rs_c] \n\t" // Load rs_c. +" lsl x14,x13,#3 \n\t" // rs_c * sizeof(double). +" \n\t" +" prfm pldl1keep,[x2,0] \n\t" // Prefetch c. +" prfm pldl1keep,[x2,x10] \n\t" // Prefetch c. +" prfm pldl1keep,[x2,x11] \n\t" // Prefetch c. +" prfm pldl1keep,[x2,x12] \n\t" // Prefetch c. " \n\t" " movi v12.2d,#0 \n\t" // Vector for accummulating column 0 " movi v13.2d,#0 \n\t" // Vector for accummulating column 0 @@ -489,10 +487,33 @@ __asm__ volatile " movi v18.2d,#0 \n\t" // Vector for accummulating column 3 " movi v19.2d,#0 \n\t" // Vector for accummulating column 3 " \n\t" +" movi v20.2d,#0 \n\t" // Vector for accummulating column 0 +" movi v21.2d,#0 \n\t" // Vector for accummulating column 0 +" movi v22.2d,#0 \n\t" // Vector for accummulating column 1 +" movi v23.2d,#0 \n\t" // Vector for accummulating column 1 +" movi v24.2d,#0 \n\t" // Vector for accummulating column 2 +" movi v25.2d,#0 \n\t" // Vector for accummulating column 2 +" movi v26.2d,#0 \n\t" // Vector for accummulating column 3 +" movi v27.2d,#0 \n\t" // Vector for accummulating column 3 +" \n\t" " ld1r {v31.2d},[x8] \n\t" // Load beta " \n\t" +" \n\t" +" cmp x5,#0 \n\t" // If k_iter == 0, jump to k_left. +" beq .DCONSIDERKLEFT \n\t" +" \n\t" +" ldp q0,q1,[x0],32 \n\t" // Load a +" ldp q4,q5,[x1],32 \n\t" // Load b +" \n\t" +" cmp x5,1 \n\t" // If there is just one k_iter, jump to that one. +" beq .DLASTITER \n\t" // (as loop is do-while-like). +" \n\t" +" \n\t" " DLOOP: \n\t" // Body " \n\t" +" prfm pldl1keep,[x0,#1024] \n\t" // Prefetch. +" prfm pldl1keep,[x1,#1024] \n\t" // Prefetch. +" \n\t" " ldp q6,q7,[x1],32 \n\t" // Load b+4 into quad " \n\t" " fmla v12.2d,v0.2d,v4.d[0] \n\t" // Accummulate @@ -525,9 +546,86 @@ __asm__ volatile " fmla v17.2d,v3.2d,v7.d[0] \n\t" // Accummulate " fmla v19.2d,v3.2d,v7.d[1] \n\t" // Accummulate " \n\t" -" add x4,x4,1 \n\t" // i = i+1 -" cmp x4,x5 \n\t" // Continue -" blt DLOOP \n\t" // if i < N +" prfm pldl1keep,[x0,#64] \n\t" // Prefetch. +" prfm pldl1keep,[x1,#64] \n\t" // Prefetch. +" \n\t" +" sub x5,x5,1 \n\t" // i-=1 +" cmp x5,1 \n\t" // Iterate again if we are not in k_iter == 1. +"bne DLOOP \n\t" +" \n\t" +".DLASTITER: \n\t" +" \n\t" +" ldp q6,q7,[x1],32 \n\t" // Load b+4 into quad +" \n\t" +" fmla v12.2d,v0.2d,v4.d[0] \n\t" // Accummulate +" fmla v14.2d,v0.2d,v4.d[1] \n\t" // Accummulate +" \n\t" +" fmla v13.2d,v1.2d,v4.d[0] \n\t" // Accummulate +" fmla v15.2d,v1.2d,v4.d[1] \n\t" // Accummulate +" \n\t" +" ldp q2,q3,[x0],32 \n\t" // Load a+4 into quad +" \n\t" +" fmla v16.2d,v0.2d,v5.d[0] \n\t" // Accummulate +" fmla v18.2d,v0.2d,v5.d[1] \n\t" // Accummulate +" \n\t" +" fmla v17.2d,v1.2d,v5.d[0] \n\t" // Accummulate +" fmla v19.2d,v1.2d,v5.d[1] \n\t" // Accummulate +" \n\t" +" ld1r {v30.2d},[x7] \n\t" // Load alpha. +" \n\t" +" fmla v12.2d,v2.2d,v6.d[0] \n\t" // Accummulate +" fmla v14.2d,v2.2d,v6.d[1] \n\t" // Accummulate +" \n\t" +" fmla v13.2d,v3.2d,v6.d[0] \n\t" // Accummulate +" fmla v15.2d,v3.2d,v6.d[1] \n\t" // Accummulate +" \n\t" +" fmla v16.2d,v2.2d,v7.d[0] \n\t" // Accummulate +" fmla v18.2d,v2.2d,v7.d[1] \n\t" // Accummulate +" \n\t" +" fmla v17.2d,v3.2d,v7.d[0] \n\t" // Accummulate +" fmla v19.2d,v3.2d,v7.d[1] \n\t" // Accummulate +" \n\t" +" .DCONSIDERKLEFT: \n\t" +" cmp x6,0 \n\t" // If k_left == 0, we are done. +" beq .DPOSTACCUM \n\t" // else, we enter the k_left loop. +" \n\t" +".DLOOPKLEFT: \n\t" +" \n\t" +" prfm pldl1keep,[x0,#1024] \n\t" // Prefetch. +" prfm pldl1keep,[x1,#1024] \n\t" // Prefetch. +" \n\t" +" ldp q0,q1,[x0],32 \n\t" // Load a into quad +" ldp q4,q5,[x1],32 \n\t" // Load b into quad +" \n\t" +//" sub x6,x6,1 \n\t" +" \n\t" +" fmla v12.2d,v0.2d,v4.d[0] \n\t" // Accummulate +" fmla v14.2d,v0.2d,v4.d[1] \n\t" // Accummulate +" \n\t" +" fmla v13.2d,v1.2d,v4.d[0] \n\t" // Accummulate +" fmla v15.2d,v1.2d,v4.d[1] \n\t" // Accummulate +" \n\t" +" fmla v16.2d,v0.2d,v5.d[0] \n\t" // Accummulate +" fmla v18.2d,v0.2d,v5.d[1] \n\t" // Accummulate +" \n\t" +" fmla v17.2d,v1.2d,v5.d[0] \n\t" // Accummulate +" fmla v19.2d,v1.2d,v5.d[1] \n\t" // Accummulate +" \n\t" +//" cmp x6,0 \n\t" // Iterate again. +//" bne .DLOOPKLEFT \n\t" // if i!=0. +" \n\t" +" .DPOSTACCUM: \n\t" +" ld1r {v30.2d},[x7] \n\t" // Load alpha. +" \n\t" +" cmp x13,#1 \n\t" // If rs_c != 1 (column-major) +" bne .DGENSTORED \n\t" +" \n\t" +" .DCOLSTORED: \n\t" // C is column-major. +" fcmp d31,#0.0 \n\t" +" beq .DBETAZEROCOLSTORED \n\t" // Taking care of the beta==0 case. +" \n\t" +" ldr x2,%[caddr] \n\t" // Load address of C. +" \n\t" " \n\t" " ldp q0,q1,[x2] \n\t" // Load c into quad and increment by cs_c " add x2,x2,x10 \n\t" @@ -537,38 +635,140 @@ __asm__ volatile " add x2,x2,x10 \n\t" " ldp q6,q7,[x2] \n\t" // Load c into quad and increment by cs_c " \n\t" -" ld1r {v30.2d},[x7] \n\t" // Load alpha " \n\t" -" fmul v0.2d,v0.2d,v31.d[0] \n\t" // Scale by beta -" fmul v1.2d,v1.2d,v31.d[0] \n\t" // Scale by beta -" fmul v2.2d,v2.2d,v31.d[0] \n\t" // Scale by beta -" fmul v3.2d,v3.2d,v31.d[0] \n\t" // Scale by beta -" fmul v4.2d,v4.2d,v31.d[0] \n\t" // Scale by beta -" fmul v5.2d,v5.2d,v31.d[0] \n\t" // Scale by beta -" fmul v6.2d,v6.2d,v31.d[0] \n\t" // Scale by beta -" fmul v7.2d,v7.2d,v31.d[0] \n\t" // Scale by beta +" fmul v20.2d,v0.2d,v31.d[0] \n\t" // Scale by beta +" fmul v21.2d,v1.2d,v31.d[0] \n\t" // Scale by beta +" fmul v22.2d,v2.2d,v31.d[0] \n\t" // Scale by beta +" fmul v23.2d,v3.2d,v31.d[0] \n\t" // Scale by beta +" fmul v24.2d,v4.2d,v31.d[0] \n\t" // Scale by beta +" fmul v25.2d,v5.2d,v31.d[0] \n\t" // Scale by beta +" fmul v26.2d,v6.2d,v31.d[0] \n\t" // Scale by beta +" fmul v27.2d,v7.2d,v31.d[0] \n\t" // Scale by beta " \n\t" " prfm pldl2keep,[x16] \n\t" " prfm pldl2keep,[x17] \n\t" " \n\t" -" fmla v0.2d,v12.2d,v30.d[0] \n\t" // Scale by alpha -" fmla v1.2d,v13.2d,v30.d[0] \n\t" // Scale by alpha -" fmla v2.2d,v14.2d,v30.d[0] \n\t" // Scale by alpha -" fmla v3.2d,v15.2d,v30.d[0] \n\t" // Scale by alpha -" fmla v4.2d,v16.2d,v30.d[0] \n\t" // Scale by alpha -" fmla v5.2d,v17.2d,v30.d[0] \n\t" // Scale by alpha -" fmla v6.2d,v18.2d,v30.d[0] \n\t" // Scale by alpha -" fmla v7.2d,v19.2d,v30.d[0] \n\t" // Scale by alpha +" .DBETAZEROCOLSTORED: \n\t" // If beta==0, we won't read from C (nor scale). " \n\t" " ldr x2,%[caddr] \n\t" // Load address of C " \n\t" -" stp q0,q1,[x2] \n\t" // Store quad into c and increment by cs_c +" fmla v20.2d,v12.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v21.2d,v13.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v22.2d,v14.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v23.2d,v15.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v24.2d,v16.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v25.2d,v17.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v26.2d,v18.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v27.2d,v19.2d,v30.d[0] \n\t" // Scale by alpha +" \n\t" +" stp q20,q21,[x2] \n\t" // Store quad into c and increment by cs_c " add x2,x2,x10 \n\t" -" stp q2,q3,[x2] \n\t" // Store quad into c+4 and increment by cs_c +" stp q22,q23,[x2] \n\t" // Store quad into c+4 and increment by cs_c " add x2,x2,x10 \n\t" -" stp q4,q5,[x2] \n\t" // Store quad into c+8 and increment by cs_c +" stp q24,q25,[x2] \n\t" // Store quad into c+8 and increment by cs_c " add x2,x2,x10 \n\t" -" stp q6,q7,[x2] \n\t" // Store quad into c+16 and increment by cs_c +" stp q26,q27,[x2] \n\t" // Store quad into c+16 and increment by cs_c +" \n\t" +" b .DEND \n\t" // Done (TODO: this obviously needs to be moved down to remove jump). +" \n\t" +" .DGENSTORED: \n\t" // C is general-stride stored. +" \n\t" +" fcmp d31,#0.0 \n\t" +" beq .DBETAZEROGENSTORED \n\t" +" \n\t" // If beta!=0, then we can read from C. +" \n\t" // TODO: this was done fast. Rearrange to remove so many address reloads. +" ldr x2,%[caddr] \n\t" // Load address of C. +" \n\t" +" ld1 {v0.d}[0],[x2],x14 \n\t" // Load c00 into quad and increment by rs_c. +" ld1 {v0.d}[1],[x2],x14 \n\t" // Load c01 into quad and increment by rs_c. +" ld1 {v1.d}[0],[x2],x14 \n\t" // Load c02 into quad and increment by rs_c. +" ld1 {v1.d}[1],[x2],x14 \n\t" // Load c03 into quad and increment by rs_c. +" \n\t" +" ldr x2,%[caddr] \n\t" // Load address of C. +" add x2,x2,x10 \n\t" // c += cs_c. +" \n\t" +" ld1 {v2.d}[0],[x2],x14 \n\t" // Load c10 into quad and increment by rs_c. +" ld1 {v2.d}[1],[x2],x14 \n\t" // Load c11 into quad and increment by rs_c. +" ld1 {v3.d}[0],[x2],x14 \n\t" // Load c12 into quad and increment by rs_c. +" ld1 {v3.d}[1],[x2],x14 \n\t" // Load c13 into quad and increment by rs_c. +" \n\t" +" ldr x2,%[caddr] \n\t" // Load address of C. +" add x2,x2,x10 \n\t" // c += cs_c. +" add x2,x2,x10 \n\t" // c += cs_c. +" \n\t" +" ld1 {v4.d}[0],[x2],x14 \n\t" // Load c10 into quad and increment by rs_c. +" ld1 {v4.d}[1],[x2],x14 \n\t" // Load c11 into quad and increment by rs_c. +" ld1 {v5.d}[0],[x2],x14 \n\t" // Load c12 into quad and increment by rs_c. +" ld1 {v5.d}[1],[x2],x14 \n\t" // Load c13 into quad and increment by rs_c. +" \n\t" +" ldr x2,%[caddr] \n\t" // Load address of C. +" add x2,x2,x10 \n\t" // c += cs_c. +" add x2,x2,x10 \n\t" // c += cs_c. +" add x2,x2,x10 \n\t" // c += cs_c. +" \n\t" +" ld1 {v6.d}[0],[x2],x14 \n\t" // Load c10 into quad and increment by rs_c. +" ld1 {v6.d}[1],[x2],x14 \n\t" // Load c11 into quad and increment by rs_c. +" ld1 {v7.d}[0],[x2],x14 \n\t" // Load c12 into quad and increment by rs_c. +" ld1 {v7.d}[1],[x2],x14 \n\t" // Load c13 into quad and increment by rs_c. +" \n\t" +" prfm pldl1keep,[x16,0] \n\t" // Prefetch. +" prfm pldl1keep,[x17,0] \n\t" // Prefetch. +" \n\t" +" fmul v20.2d,v0.2d,v31.d[0] \n\t" // Scale by beta +" fmul v21.2d,v1.2d,v31.d[0] \n\t" // Scale by beta +" fmul v22.2d,v2.2d,v31.d[0] \n\t" // Scale by beta +" fmul v23.2d,v3.2d,v31.d[0] \n\t" // Scale by beta +" fmul v24.2d,v4.2d,v31.d[0] \n\t" // Scale by beta +" fmul v25.2d,v5.2d,v31.d[0] \n\t" // Scale by beta +" fmul v26.2d,v6.2d,v31.d[0] \n\t" // Scale by beta +" fmul v27.2d,v7.2d,v31.d[0] \n\t" // Scale by beta +" \n\t" +" .DBETAZEROGENSTORED: \n\t" // If beta==0, we cannot read from C (nor scale). +" \n\t" +" fmla v20.2d,v12.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v21.2d,v13.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v22.2d,v14.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v23.2d,v15.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v24.2d,v16.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v25.2d,v17.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v26.2d,v18.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v27.2d,v19.2d,v30.d[0] \n\t" // Scale by alpha +" \n\t" +" ldr x2,%[caddr] \n\t" // Load address of C. +" \n\t" +" st1 {v20.d}[0],[x2],x14 \n\t" // Store c00 into quad and increment by rs_c. +" st1 {v20.d}[1],[x2],x14 \n\t" // Store c01 into quad and increment by rs_c. +" st1 {v21.d}[0],[x2],x14 \n\t" // Store c02 into quad and increment by rs_c. +" st1 {v21.d}[1],[x2],x14 \n\t" // Store c03 into quad and increment by rs_c. +" \n\t" +" ldr x2,%[caddr] \n\t" // Load address of C. +" add x2,x2,x10 \n\t" // c += cs_c. +" \n\t" +" st1 {v22.d}[0],[x2],x14 \n\t" // Store c10 into quad and increment by rs_c. +" st1 {v22.d}[1],[x2],x14 \n\t" // Store c11 into quad and increment by rs_c. +" st1 {v23.d}[0],[x2],x14 \n\t" // Store c12 into quad and increment by rs_c. +" st1 {v23.d}[1],[x2],x14 \n\t" // Store c13 into quad and increment by rs_c. +" \n\t" +" ldr x2,%[caddr] \n\t" // Load address of C. +" add x2,x2,x10 \n\t" // c += cs_c. +" add x2,x2,x10 \n\t" // c += cs_c. +" \n\t" +" st1 {v24.d}[0],[x2],x14 \n\t" // Store c10 into quad and increment by rs_c. +" st1 {v24.d}[1],[x2],x14 \n\t" // Store c11 into quad and increment by rs_c. +" st1 {v25.d}[0],[x2],x14 \n\t" // Store c12 into quad and increment by rs_c. +" st1 {v25.d}[1],[x2],x14 \n\t" // Store c13 into quad and increment by rs_c. +" \n\t" +" ldr x2,%[caddr] \n\t" // Load address of C. +" add x2,x2,x10 \n\t" // c += cs_c. +" add x2,x2,x10 \n\t" // c += cs_c. +" add x2,x2,x10 \n\t" // c += cs_c. +" \n\t" +" st1 {v26.d}[0],[x2],x14 \n\t" // Store c10 into quad and increment by rs_c. +" st1 {v26.d}[1],[x2],x14 \n\t" // Store c11 into quad and increment by rs_c. +" st1 {v27.d}[0],[x2],x14 \n\t" // Store c12 into quad and increment by rs_c. +" st1 {v27.d}[1],[x2],x14 \n\t" // Store c13 into quad and increment by rs_c. +" \n\t" +" .DEND: \n\t" // Done! " \n\t" :// output operands (none) :// input operands @@ -587,13 +787,15 @@ __asm__ volatile "x0","x1","x2", "x4","x5","x6", "x7","x8","x9", - "x10","x11","x12", + "x10","x11","x12","x13","x14","x16","x17", "v0","v1","v2", "v3","v4","v5", "v6","v7","v8", "v9","v10","v11", "v12","v13","v14", "v15","v16","v17","v18","v19", + "v20","v21","v22","v23", + "v24","v25","v26","v27", "v30","v31" ); From 37e55ca39bdbddaec03ad30d43e8ad2b3e549c96 Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Fri, 30 Oct 2015 18:25:04 -0500 Subject: [PATCH 29/38] Fixed obscure 3m1/4m1a bugs in trmm[3] and trsm. Details: - Fixed a family of bugs in the triangular level-3 operations for certain complex implementations (3m1 and 4m1a) that only manifest if one of the register blocksizes (PACKMR/PACKNR, actually) is odd: - Fixed incorrect imaginary stride computation in bli_packm_blk_var2() for the triangular case. - Fixed the incorrect computation of imaginary stride, as stored in the auxinfo_t struct in trmm and trsm macro-kernels. - Fixed incorrect pointer arithmetic in the trsm macro-kernels in the cases where the the register blocksize for the triangular matrix is odd. Introduced a new byte-granular pointer arithmetic macro, bli_ptr_add(), that computes the correct value. - Added cpp macro to bli_macro_defs.h for typeof() operator, defined in terms of __typeof__, which is used by bli_ptr_add() macro. - Disabled the row- vs. column-storage optimization in bli_trmm_front() for singleton problems because the inherent ambiguity of whether a scalar is row-stored or column-stored causes the wrong parameter combination code to be executed (by dumb luck of our checking for row storage first). - Added commented-out debugging lines to 3m1/4m1a and reference micro-kernels, and trsm_ll macro-kernel. --- frame/1m/packm/bli_packm_blk_var2.c | 46 ++++++++++++---- frame/1m/packm/bli_packm_struc_cxk_3mis.c | 8 --- frame/1m/packm/bli_packm_struc_cxk_4mi.c | 8 --- frame/3/trmm/bli_trmm_front.c | 3 ++ frame/3/trmm/bli_trmm_ll_ker_var2.c | 12 +++-- frame/3/trmm/bli_trmm_lu_ker_var2.c | 12 +++-- frame/3/trmm/bli_trmm_rl_ker_var2.c | 12 +++-- frame/3/trmm/bli_trmm_ru_ker_var2.c | 12 +++-- frame/3/trsm/bli_trsm_ll_ker_var2.c | 53 +++++++++++++++++-- frame/3/trsm/bli_trsm_lu_ker_var2.c | 15 ++++-- frame/3/trsm/bli_trsm_rl_ker_var2.c | 15 ++++-- frame/3/trsm/bli_trsm_ru_ker_var2.c | 15 ++++-- .../3/trsm/ukernels/bli_gemmtrsm_l_ukr_ref.c | 7 +++ frame/include/bli_macro_defs.h | 11 ++++ frame/include/bli_param_macro_defs.h | 14 +++++ .../ukernels/trsm/bli_gemmtrsm4m1_l_ukr_ref.c | 27 +++++++++- .../ind/ukernels/trsm/bli_trsm4m1_l_ukr_ref.c | 17 ++++++ 17 files changed, 222 insertions(+), 65 deletions(-) diff --git a/frame/1m/packm/bli_packm_blk_var2.c b/frame/1m/packm/bli_packm_blk_var2.c index e8c700bb7a..c53c8e6bcc 100644 --- a/frame/1m/packm/bli_packm_blk_var2.c +++ b/frame/1m/packm/bli_packm_blk_var2.c @@ -256,6 +256,7 @@ void PASTEMAC(ch,varname)( \ conj_t conjc; \ bool_t row_stored; \ bool_t col_stored; \ + inc_t is_p_use; \ dim_t ss_num; \ dim_t ss_den; \ \ @@ -427,6 +428,14 @@ PASTEMAC(ch,fprintm)( stdout, "packm_var2: a", m, n, \ \ c_use = c_begin + (panel_off_i )*ldc; \ p_use = p_begin; \ +\ + /* We need to re-compute the imaginary stride as a function of + panel_len_max_i since triangular packed matrices have panels + of varying lengths. */ \ + is_p_use = ldp * panel_len_max_i; \ +\ + /* We nudge the imaginary stride up by one if it is odd. */ \ + is_p_use += ( bli_is_odd( is_p_use ) ? 1 : 0 ); \ \ if( packm_thread_my_iter( it, thread ) ) \ { \ @@ -444,18 +453,17 @@ PASTEMAC(ch,fprintm)( stdout, "packm_var2: a", m, n, \ kappa_cast, \ c_use, rs_c, cs_c, \ p_use, rs_p, cs_p, \ - is_p ); \ + is_p_use ); \ } \ \ /* NOTE: This value is usually LESS than ps_p because triangular matrices usually have several micro-panels that are shorter than a "full" micro-panel. */ \ +/* p_inc = ldp * panel_len_max_i; \ -\ - /* We nudge the panel increment up by one if it is odd. */ \ p_inc += ( bli_is_odd( p_inc ) ? 1 : 0 ); \ -\ - p_inc = ( p_inc * ss_num ) / ss_den; \ +*/ \ + p_inc = ( is_p_use * ss_num ) / ss_den; \ } \ else if ( bli_is_herm_or_symm( strucc ) ) \ { \ @@ -468,6 +476,8 @@ PASTEMAC(ch,fprintm)( stdout, "packm_var2: a", m, n, \ \ panel_len_i = panel_len_full; \ panel_len_max_i = panel_len_max; \ +\ + is_p_use = is_p; \ \ if( packm_thread_my_iter( it, thread ) ) \ { \ @@ -485,7 +495,7 @@ PASTEMAC(ch,fprintm)( stdout, "packm_var2: a", m, n, \ kappa_cast, \ c_use, rs_c, cs_c, \ p_use, rs_p, cs_p, \ - is_p ); \ + is_p_use ); \ } \ \ /* NOTE: This value is equivalent to ps_p. */ \ @@ -502,6 +512,8 @@ PASTEMAC(ch,fprintm)( stdout, "packm_var2: a", m, n, \ \ panel_len_i = panel_len_full; \ panel_len_max_i = panel_len_max; \ +\ + is_p_use = is_p; \ \ if( packm_thread_my_iter( it, thread ) ) \ { \ @@ -519,7 +531,7 @@ PASTEMAC(ch,fprintm)( stdout, "packm_var2: a", m, n, \ kappa_cast, \ c_use, rs_c, cs_c, \ p_use, rs_p, cs_p, \ - is_p ); \ + is_p_use ); \ } \ \ /* NOTE: This value is equivalent to ps_p. */ \ @@ -527,21 +539,33 @@ PASTEMAC(ch,fprintm)( stdout, "packm_var2: a", m, n, \ } \ \ /* - if ( bli_is_ro_packed( schema ) ) { \ + if ( bli_is_4mi_packed( schema ) ) { \ + printf( "packm_var2: is_p_use = %lu\n", is_p_use ); \ if ( col_stored ) { \ + if ( 0 ) \ PASTEMAC(chr,fprintm)( stdout, "packm_var2: a_r", *m_panel_use, *n_panel_use, \ ( ctype_r* )c_use, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ PASTEMAC(chr,fprintm)( stdout, "packm_var2: ap_r", *m_panel_max, *n_panel_max, \ - ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ + ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: ap_i", *m_panel_max, *n_panel_max, \ + ( ctype_r* )p_use + is_p_use, rs_p, cs_p, "%4.1f", "" ); \ } \ - if ( row_stored && *n_panel_use == 3 ) { \ + if ( row_stored ) { \ + if ( 0 ) \ PASTEMAC(chr,fprintm)( stdout, "packm_var2: b_r", *m_panel_use, *n_panel_use, \ ( ctype_r* )c_use, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ PASTEMAC(chr,fprintm)( stdout, "packm_var2: bp_r", *m_panel_max, *n_panel_max, \ - ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ + ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: bp_i", *m_panel_max, *n_panel_max, \ + ( ctype_r* )p_use + is_p_use, rs_p, cs_p, "%4.1f", "" ); \ } \ } \ */ \ +/* +*/ \ +\ +/* +*/ \ /* PASTEMAC(chr,fprintm)( stdout, "packm_var2: bp_rpi", *m_panel_max, *n_panel_max, \ ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ diff --git a/frame/1m/packm/bli_packm_struc_cxk_3mis.c b/frame/1m/packm/bli_packm_struc_cxk_3mis.c index 48de808d55..1ad507ec68 100644 --- a/frame/1m/packm/bli_packm_struc_cxk_3mis.c +++ b/frame/1m/packm/bli_packm_struc_cxk_3mis.c @@ -57,7 +57,6 @@ void PASTEMAC(ch,varname)( \ { \ dim_t panel_dim; \ dim_t panel_len; \ - dim_t panel_len_max; \ inc_t incc, ldc; \ inc_t ldp; \ \ @@ -69,7 +68,6 @@ void PASTEMAC(ch,varname)( \ /* Prepare to pack to row-stored column panel. */ \ panel_dim = n_panel; \ panel_len = m_panel; \ - panel_len_max = m_panel_max; \ incc = cs_c; \ ldc = rs_c; \ ldp = rs_p; \ @@ -79,7 +77,6 @@ void PASTEMAC(ch,varname)( \ /* Prepare to pack to column-stored row panel. */ \ panel_dim = m_panel; \ panel_len = n_panel; \ - panel_len_max = n_panel_max; \ incc = rs_c; \ ldc = cs_c; \ ldp = cs_p; \ @@ -122,11 +119,6 @@ void PASTEMAC(ch,varname)( \ } \ else /* ( bli_is_triangular( strucc ) ) */ \ { \ - /* We need to re-compute the imaginary stride as a function of - panel_len_max since triangular packed matrices have panels - of varying lengths. */ \ - is_p = ldp * panel_len_max; \ -\ /* Call a helper function for micro-panels of triangular matrices. */ \ PASTEMAC(ch,packm_tri_cxk_3mis)( strucc, \ diff --git a/frame/1m/packm/bli_packm_struc_cxk_4mi.c b/frame/1m/packm/bli_packm_struc_cxk_4mi.c index aa5babfe73..e84ff23e4b 100644 --- a/frame/1m/packm/bli_packm_struc_cxk_4mi.c +++ b/frame/1m/packm/bli_packm_struc_cxk_4mi.c @@ -57,7 +57,6 @@ void PASTEMAC(ch,varname)( \ { \ dim_t panel_dim; \ dim_t panel_len; \ - dim_t panel_len_max; \ inc_t incc, ldc; \ inc_t ldp; \ \ @@ -69,7 +68,6 @@ void PASTEMAC(ch,varname)( \ /* Prepare to pack to row-stored column panel. */ \ panel_dim = n_panel; \ panel_len = m_panel; \ - panel_len_max = m_panel_max; \ incc = cs_c; \ ldc = rs_c; \ ldp = rs_p; \ @@ -79,7 +77,6 @@ void PASTEMAC(ch,varname)( \ /* Prepare to pack to column-stored row panel. */ \ panel_dim = m_panel; \ panel_len = n_panel; \ - panel_len_max = n_panel_max; \ incc = rs_c; \ ldc = cs_c; \ ldp = cs_p; \ @@ -122,11 +119,6 @@ void PASTEMAC(ch,varname)( \ } \ else /* ( bli_is_triangular( strucc ) ) */ \ { \ - /* We need to re-compute the imaginary stride as a function of - panel_len_max since triangular packed matrices have panels - of varying lengths. */ \ - is_p = ldp * panel_len_max; \ -\ /* Call a helper function for micro-panels of triangular matrices. */ \ PASTEMAC(ch,packm_tri_cxk_4mi)( strucc, \ diff --git a/frame/3/trmm/bli_trmm_front.c b/frame/3/trmm/bli_trmm_front.c index 67594a1c23..70bf338f2a 100644 --- a/frame/3/trmm/bli_trmm_front.c +++ b/frame/3/trmm/bli_trmm_front.c @@ -96,6 +96,9 @@ void bli_trmm_front( side_t side, // contiguous columns, or if C is stored by columns and the micro-kernel // prefers contiguous rows, transpose the entire operation to allow the // micro-kernel to access elements of C in its preferred manner. + // NOTE: We disable the optimization for 1x1 matrices since the concept + // of row- vs. column storage breaks down. + if ( !bli_obj_is_1x1( c_local ) ) if ( ( bli_obj_is_row_stored( c_local ) && bli_func_prefers_contig_cols( bli_obj_datatype( c_local ), diff --git a/frame/3/trmm/bli_trmm_ll_ker_var2.c b/frame/3/trmm/bli_trmm_ll_ker_var2.c index acb89dc9dd..a50b05dc49 100644 --- a/frame/3/trmm/bli_trmm_ll_ker_var2.c +++ b/frame/3/trmm/bli_trmm_ll_ker_var2.c @@ -198,6 +198,7 @@ void PASTEMAC(ch,varname)( \ inc_t ss_a_num; \ inc_t ss_a_den; \ inc_t ps_a_cur; \ + inc_t is_a_cur; \ auxinfo_t aux; \ \ /* @@ -297,6 +298,9 @@ void PASTEMAC(ch,varname)( \ \ istep_a = PACKMR * k; \ istep_b = PACKNR * k_full; \ +\ + if ( bli_is_odd( istep_a ) ) istep_a += 1; \ + if ( bli_is_odd( istep_b ) ) istep_b += 1; \ \ /* Save the pack schemas of A and B to the auxinfo_t object. */ \ bli_auxinfo_set_schema_a( schema_a, aux ); \ @@ -353,9 +357,9 @@ void PASTEMAC(ch,varname)( \ \ /* Compute the panel stride for the current diagonal- intersecting micro-panel. */ \ - ps_a_cur = k_a1011 * PACKMR; \ - ps_a_cur += ( bli_is_odd( ps_a_cur ) ? 1 : 0 ); \ - ps_a_cur = ( ps_a_cur * ss_a_num ) / ss_a_den; \ + is_a_cur = k_a1011 * PACKMR; \ + is_a_cur += ( bli_is_odd( is_a_cur ) ? 1 : 0 ); \ + ps_a_cur = ( is_a_cur * ss_a_num ) / ss_a_den; \ \ if ( trmm_l_ir_my_iter( i, ir_thread ) ) { \ \ @@ -378,7 +382,7 @@ void PASTEMAC(ch,varname)( \ \ /* Save the 4m1/3m1 imaginary stride of A to the auxinfo_t object. */ \ - bli_auxinfo_set_is_a( PACKMR * k_a1011, aux ); \ + bli_auxinfo_set_is_a( is_a_cur, aux ); \ \ /* Handle interior and edge cases separately. */ \ if ( m_cur == MR && n_cur == NR ) \ diff --git a/frame/3/trmm/bli_trmm_lu_ker_var2.c b/frame/3/trmm/bli_trmm_lu_ker_var2.c index 2b23fbbab6..35a2cefe2d 100644 --- a/frame/3/trmm/bli_trmm_lu_ker_var2.c +++ b/frame/3/trmm/bli_trmm_lu_ker_var2.c @@ -198,6 +198,7 @@ void PASTEMAC(ch,varname)( \ inc_t ss_a_num; \ inc_t ss_a_den; \ inc_t ps_a_cur; \ + inc_t is_a_cur; \ auxinfo_t aux; \ \ /* @@ -304,6 +305,9 @@ void PASTEMAC(ch,varname)( \ \ istep_a = PACKMR * k; \ istep_b = PACKNR * k_full; \ +\ + if ( bli_is_odd( istep_a ) ) istep_a += 1; \ + if ( bli_is_odd( istep_b ) ) istep_b += 1; \ \ /* Save the pack schemas of A and B to the auxinfo_t object. */ \ bli_auxinfo_set_schema_a( schema_a, aux ); \ @@ -360,9 +364,9 @@ void PASTEMAC(ch,varname)( \ \ /* Compute the panel stride for the current diagonal- intersecting micro-panel. */ \ - ps_a_cur = k_a1112 * PACKMR; \ - ps_a_cur += ( bli_is_odd( ps_a_cur ) ? 1 : 0 ); \ - ps_a_cur = ( ps_a_cur * ss_a_num ) / ss_a_den; \ + is_a_cur = k_a1112 * PACKMR; \ + is_a_cur += ( bli_is_odd( is_a_cur ) ? 1 : 0 ); \ + ps_a_cur = ( is_a_cur * ss_a_num ) / ss_a_den; \ \ if ( trmm_l_ir_my_iter( i, ir_thread ) ) { \ \ @@ -385,7 +389,7 @@ void PASTEMAC(ch,varname)( \ \ /* Save the 4m1/3m1 imaginary stride of A to the auxinfo_t object. */ \ - bli_auxinfo_set_is_a( PACKMR * k_a1112, aux ); \ + bli_auxinfo_set_is_a( is_a_cur, aux ); \ \ /* Handle interior and edge cases separately. */ \ if ( m_cur == MR && n_cur == NR ) \ diff --git a/frame/3/trmm/bli_trmm_rl_ker_var2.c b/frame/3/trmm/bli_trmm_rl_ker_var2.c index 550c9a232e..941f7a7f28 100644 --- a/frame/3/trmm/bli_trmm_rl_ker_var2.c +++ b/frame/3/trmm/bli_trmm_rl_ker_var2.c @@ -198,6 +198,7 @@ void PASTEMAC(ch,varname)( \ inc_t ss_b_num; \ inc_t ss_b_den; \ inc_t ps_b_cur; \ + inc_t is_b_cur; \ auxinfo_t aux; \ \ /* @@ -304,6 +305,9 @@ void PASTEMAC(ch,varname)( \ \ istep_a = PACKMR * k_full; \ istep_b = PACKNR * k; \ +\ + if ( bli_is_odd( istep_a ) ) istep_a += 1; \ + if ( bli_is_odd( istep_b ) ) istep_b += 1; \ \ /* Save the pack schemas of A and B to the auxinfo_t object. */ \ bli_auxinfo_set_schema_a( schema_a, aux ); \ @@ -350,15 +354,15 @@ void PASTEMAC(ch,varname)( \ { \ /* Compute the panel stride for the current diagonal- intersecting micro-panel. */ \ - ps_b_cur = k_b1121 * PACKNR; \ - ps_b_cur += ( bli_is_odd( ps_b_cur ) ? 1 : 0 ); \ - ps_b_cur = ( ps_b_cur * ss_b_num ) / ss_b_den; \ + is_b_cur = k_b1121 * PACKNR; \ + is_b_cur += ( bli_is_odd( is_b_cur ) ? 1 : 0 ); \ + ps_b_cur = ( is_b_cur * ss_b_num ) / ss_b_den; \ \ if ( trmm_r_jr_my_iter( j, jr_thread ) ) { \ \ /* Save the 4m1/3m1 imaginary stride of B to the auxinfo_t object. */ \ - bli_auxinfo_set_is_b( PACKNR * k_b1121, aux ); \ + bli_auxinfo_set_is_b( is_b_cur, aux ); \ \ /* Loop over the m dimension (MR rows at a time). */ \ for ( i = 0; i < m_iter; ++i ) \ diff --git a/frame/3/trmm/bli_trmm_ru_ker_var2.c b/frame/3/trmm/bli_trmm_ru_ker_var2.c index 3e4d8dabc5..6d7127f6fe 100644 --- a/frame/3/trmm/bli_trmm_ru_ker_var2.c +++ b/frame/3/trmm/bli_trmm_ru_ker_var2.c @@ -198,6 +198,7 @@ void PASTEMAC(ch,varname)( \ inc_t ss_b_num; \ inc_t ss_b_den; \ inc_t ps_b_cur; \ + inc_t is_b_cur; \ auxinfo_t aux; \ \ /* @@ -305,6 +306,9 @@ void PASTEMAC(ch,varname)( \ \ istep_a = PACKMR * k_full; \ istep_b = PACKNR * k; \ +\ + if ( bli_is_odd( istep_a ) ) istep_a += 1; \ + if ( bli_is_odd( istep_b ) ) istep_b += 1; \ \ /* Save the pack schemas of A and B to the auxinfo_t object. */ \ bli_auxinfo_set_schema_a( schema_a, aux ); \ @@ -350,15 +354,15 @@ void PASTEMAC(ch,varname)( \ { \ /* Compute the panel stride for the current diagonal- intersecting micro-panel. */ \ - ps_b_cur = k_b0111 * PACKNR; \ - ps_b_cur += ( bli_is_odd( ps_b_cur ) ? 1 : 0 ); \ - ps_b_cur = ( ps_b_cur * ss_b_num ) / ss_b_den; \ + is_b_cur = k_b0111 * PACKNR; \ + is_b_cur += ( bli_is_odd( is_b_cur ) ? 1 : 0 ); \ + ps_b_cur = ( is_b_cur * ss_b_num ) / ss_b_den; \ \ if ( trmm_r_jr_my_iter( j, jr_thread ) ) { \ \ /* Save the 4m1/3m1 imaginary stride of B to the auxinfo_t object. */ \ - bli_auxinfo_set_is_b( PACKNR * k_b0111, aux ); \ + bli_auxinfo_set_is_b( is_b_cur, aux ); \ \ /* Loop over the m dimension (MR rows at a time). */ \ for ( i = 0; i < m_iter; ++i ) \ diff --git a/frame/3/trsm/bli_trsm_ll_ker_var2.c b/frame/3/trsm/bli_trsm_ll_ker_var2.c index 957f6f78aa..31384e1fa4 100644 --- a/frame/3/trsm/bli_trsm_ll_ker_var2.c +++ b/frame/3/trsm/bli_trsm_ll_ker_var2.c @@ -210,6 +210,7 @@ void PASTEMAC(ch,varname)( \ inc_t ss_a_num; \ inc_t ss_a_den; \ inc_t ps_a_cur; \ + inc_t is_a_cur; \ auxinfo_t aux; \ \ /* @@ -325,6 +326,9 @@ void PASTEMAC(ch,varname)( \ \ istep_a = PACKMR * k; \ istep_b = PACKNR * k_full; \ +\ + if ( bli_is_odd( istep_a ) ) istep_a += 1; \ + if ( bli_is_odd( istep_b ) ) istep_b += 1; \ \ /* Save the pack schemas of A and B to the auxinfo_t object. */ \ bli_auxinfo_set_schema_a( schema_a, aux ); \ @@ -382,14 +386,15 @@ void PASTEMAC(ch,varname)( \ \ /* Compute the panel stride for the current diagonal- intersecting micro-panel. */ \ - ps_a_cur = k_a1011 * PACKMR; \ - ps_a_cur += ( bli_is_odd( ps_a_cur ) ? 1 : 0 ); \ - ps_a_cur = ( ps_a_cur * ss_a_num ) / ss_a_den; \ + is_a_cur = k_a1011 * PACKMR; \ + is_a_cur += ( bli_is_odd( is_a_cur ) ? 1 : 0 ); \ + ps_a_cur = ( is_a_cur * ss_a_num ) / ss_a_den; \ \ /* Compute the addresses of the panel A10 and the triangular block A11. */ \ a10 = a1; \ - a11 = a1 + ( k_a10 * PACKMR ) / off_scl; \ + /* a11 = a1 + ( k_a10 * PACKMR ) / off_scl; */ \ + bli_ptr_add( a11, a1, k_a10 * PACKMR, off_scl ); \ \ /* Compute the addresses of the panel B01 and the block B11. */ \ @@ -414,7 +419,7 @@ void PASTEMAC(ch,varname)( \ \ /* Save the 4m1/3m1 imaginary stride of A to the auxinfo_t object. */ \ - bli_auxinfo_set_is_a( PACKMR * k_a1011, aux ); \ + bli_auxinfo_set_is_a( is_a_cur, aux ); \ \ /* Handle interior and edge cases separately. */ \ if ( m_cur == MR && n_cur == NR ) \ @@ -515,6 +520,44 @@ void PASTEMAC(ch,varname)( \ } \ \ /* +if ( bli_is_4mi_packed( schema_a ) ){ \ +PASTEMAC(d,fprintm)( stdout, "trsm4m1_ll_ker_var2: b_r before", k, n, \ + ( double* )b, rs_b, 1, "%4.1f", "" ); \ +PASTEMAC(d,fprintm)( stdout, "trsm4m1_ll_ker_var2: b_i before", k, n, \ + ( double* )b+72, rs_b, 1, "%4.1f", "" ); \ +}else{ \ +PASTEMAC(d,fprintm)( stdout, "trsmnat_ll_ker_var2: b_r before", k, n, \ + ( double* )b, 2*rs_b, 2, "%4.1f", "" ); \ +PASTEMAC(d,fprintm)( stdout, "trsmnat_ll_ker_var2: b_i before", k, n, \ + ( double* )b+1, 2*rs_b, 2, "%4.1f", "" ); \ +} \ +*/ \ +\ +/* +PASTEMAC(d,fprintm)( stdout, "trsm_ll_ker_var2: a11p_r computed", MR, MR, \ + ( double* )a11, 1, PACKMR, "%4.1f", "" ); \ +*/ \ +\ +/* +if ( bli_is_4mi_packed( schema_a ) ){ \ +PASTEMAC(d,fprintm)( stdout, "trsm4m1_ll_ker_var2: b_r after", k, n, \ + ( double* )b, rs_b, 1, "%4.1f", "" ); \ +PASTEMAC(d,fprintm)( stdout, "trsm4m1_ll_ker_var2: b_i after", k, n, \ + ( double* )b+72, rs_b, 1, "%4.1f", "" ); \ +}else{ \ +PASTEMAC(d,fprintm)( stdout, "trsmnat_ll_ker_var2: b_r after", k, n, \ + ( double* )b, 2*rs_b, 2, "%4.1f", "" ); \ +PASTEMAC(d,fprintm)( stdout, "trsmnat_ll_ker_var2: b_i after", k, n, \ + ( double* )b+1, 2*rs_b, 2, "%4.1f", "" ); \ +} \ + +PASTEMAC(d,fprintm)( stdout, "trsm_ll_ker_var2: b_r", m, n, \ + ( double* )c, 1, cs_c, "%4.1f", "" ); \ +PASTEMAC(d,fprintm)( stdout, "trsm_ll_ker_var2: b_i", m, n, \ + ( double* )c + 8*9, 1, cs_c, "%4.1f", "" ); \ +*/ \ +\ +/* PASTEMAC(ch,fprintm)( stdout, "trsm_ll_ker_var2: a1 (diag)", MR, k_a1011, a1, 1, MR, "%5.2f", "" ); \ PASTEMAC(ch,fprintm)( stdout, "trsm_ll_ker_var2: a11 (diag)", MR, MR, a11, 1, MR, "%5.2f", "" ); \ PASTEMAC(ch,fprintm)( stdout, "trsm_ll_ker_var2: b1 (diag)", k_a1011, NR, bp_i, NR, 1, "%5.2f", "" ); \ diff --git a/frame/3/trsm/bli_trsm_lu_ker_var2.c b/frame/3/trsm/bli_trsm_lu_ker_var2.c index 9270edda6a..0e61291340 100644 --- a/frame/3/trsm/bli_trsm_lu_ker_var2.c +++ b/frame/3/trsm/bli_trsm_lu_ker_var2.c @@ -211,6 +211,7 @@ void PASTEMAC(ch,varname)( \ inc_t ss_a_num; \ inc_t ss_a_den; \ inc_t ps_a_cur; \ + inc_t is_a_cur; \ auxinfo_t aux; \ \ /* @@ -333,6 +334,9 @@ void PASTEMAC(ch,varname)( \ \ istep_a = PACKMR * k; \ istep_b = PACKNR * k_full; \ +\ + if ( bli_is_odd( istep_a ) ) istep_a += 1; \ + if ( bli_is_odd( istep_b ) ) istep_b += 1; \ \ /* Save the pack schemas of A and B to the auxinfo_t object. */ \ bli_auxinfo_set_schema_a( schema_a, aux ); \ @@ -392,14 +396,15 @@ void PASTEMAC(ch,varname)( \ \ /* Compute the panel stride for the current diagonal- intersecting micro-panel. */ \ - ps_a_cur = k_a1112 * PACKMR; \ - ps_a_cur += ( bli_is_odd( ps_a_cur ) ? 1 : 0 ); \ - ps_a_cur = ( ps_a_cur * ss_a_num ) / ss_a_den; \ + is_a_cur = k_a1112 * PACKMR; \ + is_a_cur += ( bli_is_odd( is_a_cur ) ? 1 : 0 ); \ + ps_a_cur = ( is_a_cur * ss_a_num ) / ss_a_den; \ \ /* Compute the addresses of the triangular block A11 and the panel A12. */ \ a11 = a1; \ - a12 = a1 + ( k_a11 * PACKMR ) / off_scl; \ + /* a12 = a1 + ( k_a11 * PACKMR ) / off_scl; */ \ + bli_ptr_add( a12, a1, k_a11 * PACKMR, off_scl ); \ \ /* Compute the addresses of the panel B01 and the block B11. */ \ @@ -424,7 +429,7 @@ void PASTEMAC(ch,varname)( \ \ /* Save the 4m1/3m1 imaginary stride of A to the auxinfo_t object. */ \ - bli_auxinfo_set_is_a( PACKMR * k_a1112, aux ); \ + bli_auxinfo_set_is_a( is_a_cur, aux ); \ \ /* Handle interior and edge cases separately. */ \ if ( m_cur == MR && n_cur == NR ) \ diff --git a/frame/3/trsm/bli_trsm_rl_ker_var2.c b/frame/3/trsm/bli_trsm_rl_ker_var2.c index b90fd19d22..7a8e974904 100644 --- a/frame/3/trsm/bli_trsm_rl_ker_var2.c +++ b/frame/3/trsm/bli_trsm_rl_ker_var2.c @@ -211,6 +211,7 @@ void PASTEMAC(ch,varname)( \ inc_t ss_b_num; \ inc_t ss_b_den; \ inc_t ps_b_cur; \ + inc_t is_b_cur; \ auxinfo_t aux; \ \ /* @@ -346,6 +347,9 @@ void PASTEMAC(ch,varname)( \ \ istep_a = PACKMR * k_full; \ istep_b = PACKNR * k; \ +\ + if ( bli_is_odd( istep_a ) ) istep_a += 1; \ + if ( bli_is_odd( istep_b ) ) istep_b += 1; \ \ /* Save the pack schemas of A and B to the auxinfo_t object. NOTE: We swap the values for A and B since the triangular @@ -399,18 +403,19 @@ void PASTEMAC(ch,varname)( \ /* Compute the addresses of the triangular block B11 and the panel B21. */ \ b11 = b1; \ - b21 = b1 + ( k_b11 * PACKNR ) / off_scl; \ + /* b21 = b1 + ( k_b11 * PACKNR ) / off_scl; */ \ + bli_ptr_add( b21, b1, k_b11 * PACKNR, off_scl ); \ \ /* Compute the panel stride for the current micro-panel. */ \ - ps_b_cur = k_b1121 * PACKNR; \ - ps_b_cur += ( bli_is_odd( ps_b_cur ) ? 1 : 0 ); \ - ps_b_cur = ( ps_b_cur * ss_b_num ) / ss_b_den; \ + is_b_cur = k_b1121 * PACKNR; \ + is_b_cur += ( bli_is_odd( is_b_cur ) ? 1 : 0 ); \ + ps_b_cur = ( is_b_cur * ss_b_num ) / ss_b_den; \ \ /* Save the 4m1/3m1 imaginary stride of B to the auxinfo_t object. NOTE: We swap the values for A and B since the triangular "A" matrix is actually contained within B. */ \ - bli_auxinfo_set_is_a( PACKNR * k_b1121, aux ); \ + bli_auxinfo_set_is_a( is_b_cur, aux ); \ \ /* Loop over the m dimension (MR rows at a time). */ \ for ( i = 0; i < m_iter; ++i ) \ diff --git a/frame/3/trsm/bli_trsm_ru_ker_var2.c b/frame/3/trsm/bli_trsm_ru_ker_var2.c index 86187be208..bd66d654ec 100644 --- a/frame/3/trsm/bli_trsm_ru_ker_var2.c +++ b/frame/3/trsm/bli_trsm_ru_ker_var2.c @@ -210,6 +210,7 @@ void PASTEMAC(ch,varname)( \ inc_t ss_b_num; \ inc_t ss_b_den; \ inc_t ps_b_cur; \ + inc_t is_b_cur; \ auxinfo_t aux; \ \ /* @@ -341,6 +342,9 @@ void PASTEMAC(ch,varname)( \ \ istep_a = PACKMR * k_full; \ istep_b = PACKNR * k; \ +\ + if ( bli_is_odd( istep_a ) ) istep_a += 1; \ + if ( bli_is_odd( istep_b ) ) istep_b += 1; \ \ /* Save the pack schemas of A and B to the auxinfo_t object. NOTE: We swap the values for A and B since the triangular @@ -392,18 +396,19 @@ void PASTEMAC(ch,varname)( \ /* Compute the addresses of the panel B10 and the triangular block B11. */ \ b01 = b1; \ - b11 = b1 + ( k_b01 * PACKNR ) / off_scl; \ + /* b11 = b1 + ( k_b01 * PACKNR ) / off_scl; */ \ + bli_ptr_add( b11, b1, k_b01 * PACKNR, off_scl ); \ \ /* Compute the panel stride for the current micro-panel. */ \ - ps_b_cur = k_b0111 * PACKNR; \ - ps_b_cur += ( bli_is_odd( ps_b_cur ) ? 1 : 0 ); \ - ps_b_cur = ( ps_b_cur * ss_b_num ) / ss_b_den; \ + is_b_cur = k_b0111 * PACKNR; \ + is_b_cur += ( bli_is_odd( is_b_cur ) ? 1 : 0 ); \ + ps_b_cur = ( is_b_cur * ss_b_num ) / ss_b_den; \ \ /* Save the 4m1/3m1 imaginary stride of B to the auxinfo_t object. NOTE: We swap the values for A and B since the triangular "A" matrix is actually contained within B. */ \ - bli_auxinfo_set_is_a( PACKNR * k_b0111, aux ); \ + bli_auxinfo_set_is_a( is_b_cur, aux ); \ \ /* Loop over the m dimension (MR rows at a time). */ \ for ( i = 0; i < m_iter; ++i ) \ diff --git a/frame/3/trsm/ukernels/bli_gemmtrsm_l_ukr_ref.c b/frame/3/trsm/ukernels/bli_gemmtrsm_l_ukr_ref.c index cdd6ad4594..37f0aa3cd0 100644 --- a/frame/3/trsm/ukernels/bli_gemmtrsm_l_ukr_ref.c +++ b/frame/3/trsm/ukernels/bli_gemmtrsm_l_ukr_ref.c @@ -69,6 +69,13 @@ void PASTEMAC(ch,varname)( \ b11, \ c11, rs_c, cs_c, \ data ); \ +\ +/* +PASTEMAC(d,fprintm)( stdout, "gemmtrsm_l_ukr: b0111p_r after", k+3, 8, \ + ( double* )b01, 2*PASTEMAC(ch,packnr), 2, "%4.1f", "" ); \ +PASTEMAC(d,fprintm)( stdout, "gemmtrsm_l_ukr: b0111p_i after", k+3, 8, \ + ( double* )b01 + 1, 2*PASTEMAC(ch,packnr), 2, "%4.1f", "" ); \ +*/ \ } INSERT_GENTFUNC_BASIC2( gemmtrsm_l_ukr_ref, GEMM_UKERNEL, TRSM_L_UKERNEL ) diff --git a/frame/include/bli_macro_defs.h b/frame/include/bli_macro_defs.h index 907a07d835..95c8580027 100644 --- a/frame/include/bli_macro_defs.h +++ b/frame/include/bli_macro_defs.h @@ -49,6 +49,17 @@ #endif +// -- Define typeof() operator if using non-GNU compiler -- + +#ifndef __GNUC__ + #define typeof __typeof__ +#else + #ifndef typeof + #define typeof __typeof__ + #endif +#endif + + // -- Boolean values -- #ifndef TRUE diff --git a/frame/include/bli_param_macro_defs.h b/frame/include/bli_param_macro_defs.h index 1b1040b1fb..c2e05ccaca 100644 --- a/frame/include/bli_param_macro_defs.h +++ b/frame/include/bli_param_macro_defs.h @@ -653,6 +653,20 @@ bli_is_rpi_packed( schema ) ) +// pointer-related + +// p1 = p0 + (num/dem) +#define bli_ptr_add( p1, p0, num, dem ) \ +{ \ + p1 = ( typeof( p1 ) ) \ + ( ( char* )(p0) + ( ( (num) * sizeof( *(p0) ) \ + ) / (dem) \ + ) \ + ); \ +} + + + // return datatype for char diff --git a/frame/ind/ukernels/trsm/bli_gemmtrsm4m1_l_ukr_ref.c b/frame/ind/ukernels/trsm/bli_gemmtrsm4m1_l_ukr_ref.c index f73106485e..d4d885c43e 100644 --- a/frame/ind/ukernels/trsm/bli_gemmtrsm4m1_l_ukr_ref.c +++ b/frame/ind/ukernels/trsm/bli_gemmtrsm4m1_l_ukr_ref.c @@ -79,6 +79,17 @@ void PASTEMAC(ch,varname)( \ \ dim_t i, j; \ \ +/* +printf( "gemmtrsm4m1_l_ukr: is_a = %lu is_b = %lu\n", is_a, is_b ); \ +PASTEMAC(chr,fprintm)( stdout, "gemmtrsm4m1_l_ukr: a1011p_r", m, k+m, \ + a10_r, 1, PASTEMAC(chr,packmr), "%4.1f", "" ); \ +PASTEMAC(chr,fprintm)( stdout, "gemmtrsm4m1_l_ukr: a1011p_i", m, k+m, \ + a10_i, 1, PASTEMAC(chr,packmr), "%4.1f", "" ); \ +PASTEMAC(chr,fprintm)( stdout, "gemmtrsm4m1_l_ukr: b0111p_r", k+m, n, \ + b01_r, PASTEMAC(chr,packnr), 1, "%4.1f", "" ); \ +PASTEMAC(chr,fprintm)( stdout, "gemmtrsm4m1_l_ukr: b0111p_i", k+m, n, \ + b01_i, PASTEMAC(chr,packnr), 1, "%4.1f", "" ); \ +*/ \ \ /* Copy the contents of c to a temporary buffer ct. */ \ if ( !PASTEMAC(chr,eq0)( alpha_i ) ) \ @@ -99,7 +110,7 @@ void PASTEMAC(ch,varname)( \ \ \ /* b11.r = alpha.r * b11.r - ( a10.r * b01.r - a10.i * b01.i ); - b11.i = alpha.r * b11.r - ( a10.r * b01.i + a10.i * b01.r ); */ \ + b11.i = alpha.r * b11.i - ( a10.r * b01.i + a10.i * b01.r ); */ \ \ bli_auxinfo_set_next_ab( a10_r, b01_i, *data ); \ \ @@ -144,7 +155,12 @@ void PASTEMAC(ch,varname)( \ one_r, \ b11_r, rs_b, cs_b, \ data ); \ -\ +/* +PASTEMAC(chr,fprintm)( stdout, "gemmtrsm4m1_l_ukr: b0111p_r post-gemm", k+m, n, \ + b01_r, PASTEMAC(chr,packnr), 1, "%4.1f", "" ); \ +PASTEMAC(chr,fprintm)( stdout, "gemmtrsm4m1_l_ukr: b0111p_i post-gemm", k+m, n, \ + b01_i, PASTEMAC(chr,packnr), 1, "%4.1f", "" ); \ +*/ \ \ /* b11 = inv(a11) * b11; c11 = b11; */ \ @@ -152,6 +168,13 @@ void PASTEMAC(ch,varname)( \ b11_r, \ c11, rs_c, cs_c, \ data ); \ +\ +/* +PASTEMAC(chr,fprintm)( stdout, "gemmtrsm4m1_l_ukr: b0111p_r after", k+m, n, \ + b01_r, PASTEMAC(chr,packnr), 1, "%4.1f", "" ); \ +PASTEMAC(chr,fprintm)( stdout, "gemmtrsm4m1_l_ukr: b0111p_i after", k+m, n, \ + b01_i, PASTEMAC(chr,packnr), 1, "%4.1f", "" ); \ +*/ \ } INSERT_GENTFUNCCO_BASIC2( gemmtrsm4m1_l_ukr_ref, GEMM_UKERNEL, TRSM4M1_L_UKERNEL ) diff --git a/frame/ind/ukernels/trsm/bli_trsm4m1_l_ukr_ref.c b/frame/ind/ukernels/trsm/bli_trsm4m1_l_ukr_ref.c index d03ef40a4f..e11faabe6b 100644 --- a/frame/ind/ukernels/trsm/bli_trsm4m1_l_ukr_ref.c +++ b/frame/ind/ukernels/trsm/bli_trsm4m1_l_ukr_ref.c @@ -66,6 +66,16 @@ void PASTEMAC(ch,varname)( \ dim_t iter, i, j, l; \ dim_t n_behind; \ \ +/* +PASTEMAC(chr,fprintm)( stdout, "trsm4m1_l_ukr: a11p_r", m, m, \ + a_r, 1, PASTEMAC(chr,packmr), "%4.1f", "" ); \ +PASTEMAC(chr,fprintm)( stdout, "trsm4m1_l_ukr: a11p_i", m, m, \ + a_i, 1, PASTEMAC(chr,packmr), "%4.1f", "" ); \ +PASTEMAC(chr,fprintm)( stdout, "trsm4m1_l_ukr: b11p_r", m, n, \ + b_r, PASTEMAC(chr,packnr), 1, "%4.1f", "" ); \ +PASTEMAC(chr,fprintm)( stdout, "trsm4m1_l_ukr: b11p_i", m, n, \ + b_i, PASTEMAC(chr,packnr), 1, "%4.1f", "" ); \ +*/ \ \ for ( iter = 0; iter < m; ++iter ) \ { \ @@ -136,6 +146,13 @@ void PASTEMAC(ch,varname)( \ PASTEMAC(chr,copys)( beta11c_i, *beta11_i ); \ } \ } \ +\ +/* +PASTEMAC(chr,fprintm)( stdout, "trsm4m1_l_ukr: b11p_r after", m, n, \ + b_r, PASTEMAC(chr,packnr), 1, "%4.1f", "" ); \ +PASTEMAC(chr,fprintm)( stdout, "trsm4m1_l_ukr: b11p_i after", m, n, \ + b_i, PASTEMAC(chr,packnr), 1, "%4.1f", "" ); \ +*/ \ } INSERT_GENTFUNCCO_BASIC0( trsm4m1_l_ukr_ref ) From 4a502fbe77bd0f701108baaa559d9cfb483f88de Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Mon, 2 Nov 2015 13:28:34 -0600 Subject: [PATCH 30/38] Laid groundwork for runtime memory pool resizing. Details: - Changed bli_pool_finalize() so that the freeing begins with the block at top_index instead of block 0. This allows us to use the function for terminal finalization as well as temporary cleanup prior to reinitialization. Also, clear the pool_t struct upon _pool_finalize() in case it is called in the terminal case with some blocks still checked out to threads (in which case the threads will see the new block size as 0 and thus release the block as intended). - Added bli_pool_reinit(), which calls _pool_finalize() followed by _pool_init() with new parameters. - Added bli_mem_reinit(), which is based on bli_pool_reinit(). - Added new wrapper, _mem_compute_pool_block_sizes(), which calls _mem_compute_pool_block_sizes_dt(). - Updated bli_mem_release() so that the pblk_t is freed, via _pool_free_block(), if the block size recorded in the mem_t at the time the pblk_t was acquired is now different from the value in the pool_t. --- frame/1m/packm/bli_packm_init.c | 1 - frame/base/bli_mem.c | 181 ++++++++++++++++++++++++++------ frame/base/bli_mem.h | 5 + frame/base/bli_pool.c | 44 ++++++-- frame/base/bli_pool.h | 4 + 5 files changed, 194 insertions(+), 41 deletions(-) diff --git a/frame/1m/packm/bli_packm_init.c b/frame/1m/packm/bli_packm_init.c index caa4e48393..4c35e02015 100644 --- a/frame/1m/packm/bli_packm_init.c +++ b/frame/1m/packm/bli_packm_init.c @@ -175,7 +175,6 @@ void bli_packm_init_pack( invdiag_t invert_diag, obj_t* p ) { num_t dt = bli_obj_datatype( *c ); - num_t dt_real = bli_obj_datatype_proj_to_real( *c ); trans_t transc = bli_obj_onlytrans_status( *c ); dim_t m_c = bli_obj_length( *c ); dim_t n_c = bli_obj_width( *c ); diff --git a/frame/base/bli_mem.c b/frame/base/bli_mem.c index 81a7a2c3b6..e70fcd372c 100644 --- a/frame/base/bli_mem.c +++ b/frame/base/bli_mem.c @@ -117,6 +117,13 @@ void bli_mem_acquire_m( siz_t req_size, // the struct's pblk_t field. bli_pool_checkout_block( pblk, pool ); + // Query the size of the blocks in the pool so we can store it in + // the mem_t object. At this point, it is guaranteed to be at + // least as large as req_size. (NOTE: We must perform the query + // within the critical section to ensure that the pool hasn't + // changed, as unlikely as that would be.) + block_size = bli_pool_block_size( pool ); + } // END CRITICAL SECTION @@ -124,11 +131,6 @@ void bli_mem_acquire_m( siz_t req_size, pthread_mutex_unlock( &mem_manager_mutex ); #endif - // Query the size of the blocks in the pool so we can store it in the - // mem_t object. At this point, it is guaranteed to be at least as - // large as req_size. - block_size = bli_pool_block_size( pool ); - // Initialize the mem_t object with: // - the buffer type (a packbuf_t value), // - the address of the memory pool to which it belongs, and @@ -148,6 +150,8 @@ void bli_mem_release( mem_t* mem ) packbuf_t buf_type; pool_t* pool; pblk_t* pblk; + siz_t block_size_cur; + siz_t block_size_prev; // Make sure the API is initialized. bli_mem_init(); @@ -172,6 +176,11 @@ void bli_mem_release( mem_t* mem ) // Extract the address of the pblk_t struct within the mem_t struct. pblk = bli_mem_pblk( mem ); + // Query the size of the blocks that were in the pool at the time + // the pblk_t was checked out. (This is used below, in the critical + // section.) + block_size_prev = bli_mem_size( mem ); + #ifdef BLIS_ENABLE_OPENMP _Pragma( "omp critical (mem)" ) #endif @@ -182,8 +191,26 @@ void bli_mem_release( mem_t* mem ) // BEGIN CRITICAL SECTION { - // Check the block back into the pool. - bli_pool_checkin_block( pblk, pool ); + // Query the size of the blocks currently in the pool. + block_size_cur = bli_pool_block_size( pool ); + + // If the block size of the pool has changed since the pblk_t + // was checked out, then we need to free the pblk_t rather + // than check it back in. Why? Because the pool's block size + // has (most likely) increased to meet changing needs (example: + // larger cache blocksizes). Thus, the current pblk_t's smaller + // allocated size is of no use anymore. + if ( block_size_cur != block_size_prev ) + { + // Free the pblk_t using the appropriate function in the + // pool API. + bli_pool_free_block( pblk ); + } + else + { + // Check the block back into the pool. + bli_pool_checkin_block( pblk, pool ); + } } // END CRITICAL SECTION @@ -281,6 +308,41 @@ void bli_mem_init( void ) #endif } +void bli_mem_reinit( void ) +{ + +#ifdef BLIS_ENABLE_OPENMP + _Pragma( "omp critical (mem)" ) +#endif +#ifdef BLIS_ENABLE_PTHREADS + pthread_mutex_lock( &mem_manager_mutex ); +#endif + + // BEGIN CRITICAL SECTION + { + // If for some reason the memory pools have not yet been + // initialized (unlikely), we emulate the body of bli_mem_init(). + if ( bli_mem_is_init == FALSE ) + { + // Initialize the memory pools. + bli_mem_init_pools(); + + // After initialization, mark the API as initialized. + bli_mem_is_init = TRUE; + } + else + { + // Reinitialize the memory pools. + bli_mem_reinit_pools(); + } + } + // END CRITICAL SECTION + +#ifdef BLIS_ENABLE_PTHREADS + pthread_mutex_unlock( &mem_manager_mutex ); +#endif +} + void bli_mem_finalize( void ) { // If the initialization flag is FALSE, we know the API is already @@ -331,16 +393,81 @@ void bli_mem_init_pools( void ) dim_t index_b = bli_packbuf_index( BLIS_BUFFER_FOR_B_PANEL ); dim_t index_c = bli_packbuf_index( BLIS_BUFFER_FOR_C_PANEL ); + siz_t align_size = BLIS_POOL_ADDR_ALIGN_SIZE; + + siz_t block_size_a = 0; + siz_t block_size_b = 0; + siz_t block_size_c = 0; + + // Start with empty pools. dim_t num_blocks_a = 0; dim_t num_blocks_b = 0; dim_t num_blocks_c = 0; + // Determine the block size for each memory pool. + bli_mem_compute_pool_block_sizes( &block_size_a, + &block_size_b, + &block_size_c ); + + // Initialize the memory pools for A, B, and C. + bli_pool_init( num_blocks_a, block_size_a, align_size, &pools[ index_a ] ); + bli_pool_init( num_blocks_b, block_size_b, align_size, &pools[ index_b ] ); + bli_pool_init( num_blocks_c, block_size_c, align_size, &pools[ index_c ] ); +} + +void bli_mem_reinit_pools( void ) +{ + // Map each of the packbuf_t values to an index starting at zero. + dim_t index_a = bli_packbuf_index( BLIS_BUFFER_FOR_A_BLOCK ); + dim_t index_b = bli_packbuf_index( BLIS_BUFFER_FOR_B_PANEL ); + dim_t index_c = bli_packbuf_index( BLIS_BUFFER_FOR_C_PANEL ); + siz_t align_size = BLIS_POOL_ADDR_ALIGN_SIZE; siz_t block_size_a = 0; siz_t block_size_b = 0; siz_t block_size_c = 0; + // Query the number of blocks currently allocated in each pool. + dim_t num_blocks_a = bli_pool_num_blocks( &pools[ index_a ] ); + dim_t num_blocks_b = bli_pool_num_blocks( &pools[ index_b ] ); + dim_t num_blocks_c = bli_pool_num_blocks( &pools[ index_c ] ); + + // Determine the block size for each memory pool. + bli_mem_compute_pool_block_sizes( &block_size_a, + &block_size_b, + &block_size_c ); + + // Reinitialize the memory pools for A, B, and C with the same number + // of blocks as before, except with new block sizes. + bli_pool_reinit( num_blocks_a, block_size_a, align_size, &pools[ index_a ] ); + bli_pool_reinit( num_blocks_b, block_size_b, align_size, &pools[ index_b ] ); + bli_pool_reinit( num_blocks_c, block_size_c, align_size, &pools[ index_c ] ); +} + +void bli_mem_finalize_pools( void ) +{ + // Map each of the packbuf_t values to an index starting at zero. + dim_t index_a = bli_packbuf_index( BLIS_BUFFER_FOR_A_BLOCK ); + dim_t index_b = bli_packbuf_index( BLIS_BUFFER_FOR_B_PANEL ); + dim_t index_c = bli_packbuf_index( BLIS_BUFFER_FOR_C_PANEL ); + + // Finalize the memory pools for A, B, and C. + bli_pool_finalize( &pools[ index_a ] ); + bli_pool_finalize( &pools[ index_b ] ); + bli_pool_finalize( &pools[ index_c ] ); +} + +// ----------------------------------------------------------------------------- + +void bli_mem_compute_pool_block_sizes( siz_t* bs_a, + siz_t* bs_b, + siz_t* bs_c ) +{ + siz_t bs_cand_a = 0; + siz_t bs_cand_b = 0; + siz_t bs_cand_c = 0; + ind_t im; num_t dt; @@ -355,38 +482,24 @@ void bli_mem_init_pools( void ) // Avoid considering induced methods for real datatypes. if ( bli_is_complex( dt ) || im == BLIS_NAT ) { - siz_t bs_a, bs_b, bs_c; -//printf( "bli_mem_init_pools: considering dt %lu im %lu\n", dt, im ); + siz_t bs_dt_a, bs_dt_b, bs_dt_c; + bli_mem_compute_pool_block_sizes_dt( dt, im, - &bs_a, - &bs_b, - &bs_c ); - -//printf( "bli_mem_init_pools: bs a b c = %lu %lu %lu\n", bs_a, bs_b, bs_c ); - block_size_a = bli_max( bs_a, block_size_a ); - block_size_b = bli_max( bs_b, block_size_b ); - block_size_c = bli_max( bs_c, block_size_c ); + &bs_dt_a, + &bs_dt_b, + &bs_dt_c ); + + bs_cand_a = bli_max( bs_dt_a, bs_cand_a ); + bs_cand_b = bli_max( bs_dt_b, bs_cand_b ); + bs_cand_c = bli_max( bs_dt_c, bs_cand_c ); } } } - // Initialize the memory pools for A, B, and C. - bli_pool_init( num_blocks_a, block_size_a, align_size, &pools[ index_a ] ); - bli_pool_init( num_blocks_b, block_size_b, align_size, &pools[ index_b ] ); - bli_pool_init( num_blocks_c, block_size_c, align_size, &pools[ index_c ] ); -} - -void bli_mem_finalize_pools( void ) -{ - // Map each of the packbuf_t values to an index starting at zero. - dim_t index_a = bli_packbuf_index( BLIS_BUFFER_FOR_A_BLOCK ); - dim_t index_b = bli_packbuf_index( BLIS_BUFFER_FOR_B_PANEL ); - dim_t index_c = bli_packbuf_index( BLIS_BUFFER_FOR_C_PANEL ); - - // Finalize the memory pools for A, B, and C. - bli_pool_finalize( &pools[ index_a ] ); - bli_pool_finalize( &pools[ index_b ] ); - bli_pool_finalize( &pools[ index_c ] ); + // Save the results. + *bs_a = bs_cand_a; + *bs_b = bs_cand_b; + *bs_c = bs_cand_c; } // ----------------------------------------------------------------------------- diff --git a/frame/base/bli_mem.h b/frame/base/bli_mem.h index 59572ad41c..f80e73aa61 100644 --- a/frame/base/bli_mem.h +++ b/frame/base/bli_mem.h @@ -33,6 +33,7 @@ */ void bli_mem_init( void ); +void bli_mem_reinit( void ); void bli_mem_finalize( void ); bool_t bli_mem_is_initialized( void ); @@ -52,8 +53,12 @@ siz_t bli_mem_pool_size( packbuf_t buf_type ); // ----------------------------------------------------------------------------- void bli_mem_init_pools( void ); +void bli_mem_reinit_pools( void ); void bli_mem_finalize_pools( void ); +void bli_mem_compute_pool_block_sizes( siz_t* bs_a, + siz_t* bs_b, + siz_t* bs_c ); void bli_mem_compute_pool_block_sizes_dt( num_t dt, ind_t method, siz_t* bs_a, diff --git a/frame/base/bli_pool.c b/frame/base/bli_pool.c index 305cd32345..bae3e3556f 100644 --- a/frame/base/bli_pool.c +++ b/frame/base/bli_pool.c @@ -64,24 +64,56 @@ void bli_pool_finalize( pool_t* pool ) { pblk_t* block_ptrs; dim_t num_blocks; + dim_t top_index; dim_t i; - // NOTE: This implementation assumes that all blocks have been - // checked in by all threads. + // NOTE: This implementation assumes that either: + // - all blocks have been checked in by all threads, or + // - some subset of blocks have been checked in and the caller + // is bli_pool_reinit(). - // Query the current block_ptrs array and total number of blocks - // presently allocated. + // Query the current block_ptrs array. block_ptrs = bli_pool_block_ptrs( pool ); + + // Query the total number of blocks presently allocated. num_blocks = bli_pool_num_blocks( pool ); - // Free the individual blocks. - for ( i = 0; i < num_blocks; ++i ) + // Query the top_index of the pool. + top_index = bli_pool_top_index( pool ); + + // Free the individual blocks currently in the pool. + for ( i = top_index; i < num_blocks; ++i ) { bli_pool_free_block( &(block_ptrs[i]) ); } // Free the block_ptrs array. bli_free( block_ptrs ); + + // Clear the contents of the pool_t struct. + bli_pool_set_block_ptrs( NULL, pool ); + bli_pool_set_block_ptrs_len( 0, pool ); + bli_pool_set_num_blocks( 0, pool ); + bli_pool_set_top_index( 0, pool ); + bli_pool_set_block_size( 0, pool ); + bli_pool_set_align_size( 0, pool ); +} + +void bli_pool_reinit( dim_t num_blocks_new, + siz_t block_size_new, + siz_t align_size_new, + pool_t* pool ) +{ + // Finalize the pool as it is currently configured. If some blocks + // are still checked out to threads, those blocks are not freed + // here, and instead will be freed when the threads are ready to + // release the blocks. (This will happen because the threads will + // notice that the block size of the pool has changed.) + bli_pool_finalize( pool ); + + // Reinitialize the pool with the new parameters, in particular, + // the new block size. + bli_pool_init( num_blocks_new, block_size_new, align_size_new, pool ); } void bli_pool_checkout_block( pblk_t* block, pool_t* pool ) diff --git a/frame/base/bli_pool.h b/frame/base/bli_pool.h index 611c0feb0d..aba3c2f51c 100644 --- a/frame/base/bli_pool.h +++ b/frame/base/bli_pool.h @@ -163,6 +163,10 @@ void bli_pool_init( dim_t num_blocks_init, siz_t align_size, pool_t* pool ); void bli_pool_finalize( pool_t* pool ); +void bli_pool_reinit( dim_t num_blocks_new, + siz_t block_size, + siz_t align_size, + pool_t* pool ); void bli_pool_checkout_block( pblk_t* block, pool_t* pool ); void bli_pool_checkin_block( pblk_t* block, pool_t* pool ); From 33557ecccaf49b2569b7f3d7bcea52c2aab94c68 Mon Sep 17 00:00:00 2001 From: Jeff Hammond Date: Mon, 2 Nov 2015 12:18:43 -0800 Subject: [PATCH 31/38] add Travis CI build status icon to the README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index c6840edef0..83920d5fd0 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ ![The BLIS cat is sleeping.](http://www.cs.utexas.edu/users/field/blis_cat.png) +[![Build Status](https://travis-ci.org/flame/blis.svg?branch=master)](https://travis-ci.org/flame/blis) + Introduction ------------ From 3e116f0a2953f50b3c068759a775ad7ffae04e49 Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Mon, 2 Nov 2015 17:18:23 -0600 Subject: [PATCH 32/38] Fixed imaginary bug in quadratic partitioning code. Details: - Fixed a bug in the relatively new quadratic partitioning code that, under the right conditions, would perform sqrt() on a negative value. If the solution is imaginary, we discard it and use an alternate partition width that assumes no diagonal intersection. That alternate width is actually already computed, so, the fix was quite simple. Thanks to Devangi Parikh for reporting this bug. --- frame/base/bli_threading.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/frame/base/bli_threading.c b/frame/base/bli_threading.c index 077089dc97..b21c0448c5 100644 --- a/frame/base/bli_threading.c +++ b/frame/base/bli_threading.c @@ -449,8 +449,9 @@ dim_t bli_get_range_width_l( doff_t diagoff_j, ( void )offm_inc; ( void )offn_inc; - // Solve a quadratic equation to find the width of the current (jth) - // subpartition given the m dimension, diagonal offset, and area. + // Prepare to solve a quadratic equation to find the width of the + // current (jth) subpartition given the m dimension, diagonal offset, + // and area. // NOTE: We know that the +/- in the quadratic formula must be a + // here because we know that the desired solution (the subpartition // width) will be smaller than (m + diagoff), not larger. If you @@ -460,12 +461,18 @@ dim_t bli_get_range_width_l( doff_t diagoff_j, const double c = -0.5 * ( ( double )diagoff_j * ( ( double )diagoff_j + 1.0 ) ) - area_per_thr; - const double x = ( -b + sqrt( b * b - 4.0 * a * c ) ) / ( 2.0 * a ); - // Use the rounded solution as our width, but make sure it didn't - // round to zero. - width = ( dim_t )bli_round( x ); - if ( width == 0 ) width = 1; + // If the quadratic solution is not imaginary, round it and use that + // as our width, but make sure it didn't round to zero. Otherwise, + // discard the quadratic solution and leave width, as previously + // computed, unchanged. + if ( b * b - 4.0 * a * c >= 0 ) + { + const double x = ( -b + sqrt( b * b - 4.0 * a * c ) ) / ( 2.0 * a ); + + width = ( dim_t )bli_round( x ); + if ( width == 0 ) width = 1; + } // Adjust the width, if necessary. if ( j == 0 && handle_edge_low ) From 3e6dd11467643fbc2cb45c13cec8dd6024232833 Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Tue, 3 Nov 2015 10:30:08 -0600 Subject: [PATCH 33/38] Minor re-expression in quadratic partitioning code. Details: - Minor change to quadratic equation solution code that avoids recomputation of the sqrt() parameter when the compiler is not smart enough to perform this optimization automatically. --- frame/base/bli_threading.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frame/base/bli_threading.c b/frame/base/bli_threading.c index b21c0448c5..168679ef0f 100644 --- a/frame/base/bli_threading.c +++ b/frame/base/bli_threading.c @@ -461,14 +461,15 @@ dim_t bli_get_range_width_l( doff_t diagoff_j, const double c = -0.5 * ( ( double )diagoff_j * ( ( double )diagoff_j + 1.0 ) ) - area_per_thr; + const double r = b * b - 4.0 * a * c; // If the quadratic solution is not imaginary, round it and use that // as our width, but make sure it didn't round to zero. Otherwise, // discard the quadratic solution and leave width, as previously // computed, unchanged. - if ( b * b - 4.0 * a * c >= 0 ) + if ( r >= 0.0 ) { - const double x = ( -b + sqrt( b * b - 4.0 * a * c ) ) / ( 2.0 * a ); + const double x = ( -b + sqrt( r ) ) / ( 2.0 * a ); width = ( dim_t )bli_round( x ); if ( width == 0 ) width = 1; From 42810bbfa0b8f006ecc5128d903909ec13ea63f9 Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Thu, 12 Nov 2015 12:07:46 -0600 Subject: [PATCH 34/38] Fixed minor bugs for uncommon obj_create cases. Details: - Separated bli_adjust_strides() into _alloc() and _attach() flavors so that the latter can avoid a test performed by the former, in which the rs and cs are overridden and set to zero if either matrix dimension is zero. Actually, we also disable this overridding behavior, even for the _alloc() case, since keeping the original strides (probably) does not hurt anything. The original code has been kept commented-out, though, in case an unintended consequence is later discovered. - Fixed a typo in an error check for general stride cases where rs == cs. --- frame/base/bli_obj.c | 41 +++++++++++++++++++++++++++++------------ frame/base/bli_obj.h | 15 ++++++++++----- 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/frame/base/bli_obj.c b/frame/base/bli_obj.c index 75f2cd7e04..3395ec1b5e 100644 --- a/frame/base/bli_obj.c +++ b/frame/base/bli_obj.c @@ -127,7 +127,7 @@ void bli_obj_alloc_buffer( inc_t rs, // Adjust the strides, if needed, before doing anything else // (particularly, before doing any error checking). - bli_adjust_strides( m, n, &rs, &cs, &is ); + bli_adjust_strides_alloc( m, n, &rs, &cs, &is ); if ( bli_error_checking_is_enabled() ) bli_obj_alloc_buffer_check( rs, cs, is, obj ); @@ -183,7 +183,7 @@ void bli_obj_alloc_buffer( inc_t rs, // If the row and column strides are equal, it almost certainly // means that m == n == 1. (If that is not the case, then // something is very wrong.) - if ( m != 1 || m != 1 ) + if ( m != 1 || n != 1 ) bli_check_error_code( BLIS_NOT_YET_IMPLEMENTED ); n_elem = 1; @@ -228,7 +228,7 @@ void bli_obj_attach_buffer( void* p, n = bli_obj_width( *obj ); // Adjust the strides, if necessary. - bli_adjust_strides( m, n, &rs, &cs, &is ); + bli_adjust_strides_attach( m, n, &rs, &cs, &is ); // Notice that we wait until after strides have been adjusted to // error-check. @@ -357,11 +357,11 @@ void bli_obj_create_const_copy_of( obj_t* a, obj_t* b ) *temp_i = ( gint_t ) bli_zreal( value ); } -void bli_adjust_strides( dim_t m, - dim_t n, - inc_t* rs, - inc_t* cs, - inc_t* is ) +void bli_adjust_strides_alloc( dim_t m, + dim_t n, + inc_t* rs, + inc_t* cs, + inc_t* is ) { // Here, we check the strides that were input from the user and modify // them if needed. @@ -370,13 +370,30 @@ void bli_adjust_strides( dim_t m, // we set row and column strides to zero. if ( m == 0 || n == 0 ) { - *rs = 0; - *cs = 0; - *is = 1; + // NOTE: Previously, we forced the row and column strides to be zero + // if the matrices was being created with a zero dimension. It is + // *probably* the case that this is not necessary. However, in case + // a problem turns up later on, I'm leaving these lines commented + // out. + //*rs = 0; + //*cs = 0; + //*is = 1; return; } - + + bli_adjust_strides_attach( m, n, rs, cs, is ); +} + +void bli_adjust_strides_attach( dim_t m, + dim_t n, + inc_t* rs, + inc_t* cs, + inc_t* is ) +{ + // Here, we check the strides that were input from the user and modify + // them if needed. + // Interpret rs = cs = 0 as request for column storage. if ( *rs == 0 && *cs == 0 && ( *is == 0 || *is == 1 ) ) { diff --git a/frame/base/bli_obj.h b/frame/base/bli_obj.h index 316024012e..d9926a52d8 100644 --- a/frame/base/bli_obj.h +++ b/frame/base/bli_obj.h @@ -78,11 +78,16 @@ void bli_obj_create_const( double value, obj_t* obj ); void bli_obj_create_const_copy_of( obj_t* a, obj_t* b ); -void bli_adjust_strides( dim_t m, - dim_t n, - inc_t* rs, - inc_t* cs, - inc_t* is ); +void bli_adjust_strides_alloc( dim_t m, + dim_t n, + inc_t* rs, + inc_t* cs, + inc_t* is ); +void bli_adjust_strides_attach( dim_t m, + dim_t n, + inc_t* rs, + inc_t* cs, + inc_t* is ); siz_t bli_datatype_size( num_t dt ); From f0a4f41b5acf55b41707ec821c4c5f9076dfbc24 Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Thu, 12 Nov 2015 15:22:50 -0600 Subject: [PATCH 35/38] Fixed unimplemented case in core2 sgemm ukernel. Details: - Implemented the "beta == 0" case for general stride output for the dunnington sgemm micro-kernel. This case had been, up until now, identical to the "beta != 0" case, which does not work when the output matrix has nan's and inf's. It had manifested as nan residuals in the test suite for right-side tests of ctrsm4m1a. Thanks to Devin Matthews for reporting this bug. --- .../x86_64/core2-sse3/3/bli_gemm_opt_d4x4.c | 163 +++++++++++++++--- 1 file changed, 137 insertions(+), 26 deletions(-) diff --git a/kernels/x86_64/core2-sse3/3/bli_gemm_opt_d4x4.c b/kernels/x86_64/core2-sse3/3/bli_gemm_opt_d4x4.c index 8b2c3974ff..b4dfb1ce9b 100644 --- a/kernels/x86_64/core2-sse3/3/bli_gemm_opt_d4x4.c +++ b/kernels/x86_64/core2-sse3/3/bli_gemm_opt_d4x4.c @@ -73,23 +73,23 @@ void bli_sgemm_opt_8x4( " \n\t" "prefetcht2 0 * 4(%%r9) \n\t" // prefetch b_next " \n\t" - "xorpd %%xmm3, %%xmm3 \n\t" - "xorpd %%xmm4, %%xmm4 \n\t" - "xorpd %%xmm5, %%xmm5 \n\t" - "xorpd %%xmm6, %%xmm6 \n\t" - " \n\t" - "prefetcht0 6 * 4(%%rcx) \n\t" // prefetch c + 0*cs_c - "xorpd %%xmm8, %%xmm8 \n\t" - "movaps %%xmm8, %%xmm9 \n\t" - "prefetcht0 6 * 4(%%rcx,%%rdi) \n\t" // prefetch c + 1*cs_c - "movaps %%xmm8, %%xmm10 \n\t" - "movaps %%xmm8, %%xmm11 \n\t" - "prefetcht0 6 * 4(%%r10) \n\t" // prefetch c + 2*cs_c - "movaps %%xmm8, %%xmm12 \n\t" - "movaps %%xmm8, %%xmm13 \n\t" - "prefetcht0 6 * 4(%%r10,%%rdi) \n\t" // prefetch c + 3*cs_c - "movaps %%xmm8, %%xmm14 \n\t" - "movaps %%xmm8, %%xmm15 \n\t" + "xorps %%xmm3, %%xmm3 \n\t" + "xorps %%xmm4, %%xmm4 \n\t" + "xorps %%xmm5, %%xmm5 \n\t" + "xorps %%xmm6, %%xmm6 \n\t" + " \n\t" + "prefetcht2 6 * 4(%%rcx) \n\t" // prefetch c + 0*cs_c + "xorps %%xmm8, %%xmm8 \n\t" + "xorps %%xmm9, %%xmm9 \n\t" + "prefetcht2 6 * 4(%%rcx,%%rdi) \n\t" // prefetch c + 1*cs_c + "xorps %%xmm10, %%xmm10 \n\t" + "xorps %%xmm11, %%xmm11 \n\t" + "prefetcht2 6 * 4(%%r10) \n\t" // prefetch c + 2*cs_c + "xorps %%xmm12, %%xmm12 \n\t" + "xorps %%xmm13, %%xmm13 \n\t" + "prefetcht2 6 * 4(%%r10,%%rdi) \n\t" // prefetch c + 3*cs_c + "xorps %%xmm14, %%xmm14 \n\t" + "xorps %%xmm15, %%xmm15 \n\t" " \n\t" " \n\t" " \n\t" @@ -646,8 +646,119 @@ void bli_sgemm_opt_8x4( " \n\t" ".SGENSTORBZ: \n\t" " \n\t" - "jmp .SGENSTORED \n\t" // use gen-stored beta != 0 case for now - //"jmp .SDONE \n\t" // jump to end. + "mulps %%xmm6, %%xmm8 \n\t" // scale by alpha, + "movaps %%xmm8, %%xmm0 \n\t" + " \n\t" + "movss %%xmm0, (%%rcx ) \n\t" // and store back to memory. + "pshufd $0x39, %%xmm0, %%xmm1 \n\t" + "movss %%xmm1, (%%rcx,%%rsi,1) \n\t" + "pshufd $0x39, %%xmm1, %%xmm2 \n\t" + "movss %%xmm2, (%%rcx,%%rsi,2) \n\t" + "pshufd $0x39, %%xmm2, %%xmm3 \n\t" + "movss %%xmm3, (%%rcx,%%r11 ) \n\t" + " \n\t" + "addq %%rdi, %%rcx \n\t" + " \n\t" + " \n\t" + "mulps %%xmm6, %%xmm12 \n\t" // scale by alpha, + "movaps %%xmm12, %%xmm0 \n\t" + " \n\t" + "movss %%xmm0, (%%rdx ) \n\t" // and store back to memory. + "pshufd $0x39, %%xmm0, %%xmm1 \n\t" + "movss %%xmm1, (%%rdx,%%rsi,1) \n\t" + "pshufd $0x39, %%xmm1, %%xmm2 \n\t" + "movss %%xmm2, (%%rdx,%%rsi,2) \n\t" + "pshufd $0x39, %%xmm2, %%xmm3 \n\t" + "movss %%xmm3, (%%rdx,%%r11 ) \n\t" + " \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "mulps %%xmm6, %%xmm9 \n\t" // scale by alpha, + "movaps %%xmm9, %%xmm0 \n\t" + " \n\t" + "movss %%xmm0, (%%rcx ) \n\t" // and store back to memory. + "pshufd $0x39, %%xmm0, %%xmm1 \n\t" + "movss %%xmm1, (%%rcx,%%rsi,1) \n\t" + "pshufd $0x39, %%xmm1, %%xmm2 \n\t" + "movss %%xmm2, (%%rcx,%%rsi,2) \n\t" + "pshufd $0x39, %%xmm2, %%xmm3 \n\t" + "movss %%xmm3, (%%rcx,%%r11 ) \n\t" + " \n\t" + "addq %%rdi, %%rcx \n\t" + " \n\t" + " \n\t" + "mulps %%xmm6, %%xmm13 \n\t" // scale by alpha, + "movaps %%xmm13, %%xmm0 \n\t" + " \n\t" + "movss %%xmm0, (%%rdx ) \n\t" // and store back to memory. + "pshufd $0x39, %%xmm0, %%xmm1 \n\t" + "movss %%xmm1, (%%rdx,%%rsi,1) \n\t" + "pshufd $0x39, %%xmm1, %%xmm2 \n\t" + "movss %%xmm2, (%%rdx,%%rsi,2) \n\t" + "pshufd $0x39, %%xmm2, %%xmm3 \n\t" + "movss %%xmm3, (%%rdx,%%r11 ) \n\t" + " \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "mulps %%xmm6, %%xmm10 \n\t" // scale by alpha, + "movaps %%xmm10, %%xmm0 \n\t" + " \n\t" + "movss %%xmm0, (%%rcx ) \n\t" // and store back to memory. + "pshufd $0x39, %%xmm0, %%xmm1 \n\t" + "movss %%xmm1, (%%rcx,%%rsi,1) \n\t" + "pshufd $0x39, %%xmm1, %%xmm2 \n\t" + "movss %%xmm2, (%%rcx,%%rsi,2) \n\t" + "pshufd $0x39, %%xmm2, %%xmm3 \n\t" + "movss %%xmm3, (%%rcx,%%r11 ) \n\t" + " \n\t" + "addq %%rdi, %%rcx \n\t" + " \n\t" + " \n\t" + "mulps %%xmm6, %%xmm14 \n\t" // scale by alpha, + "movaps %%xmm14, %%xmm0 \n\t" + " \n\t" + "movss %%xmm0, (%%rdx ) \n\t" // and store back to memory. + "pshufd $0x39, %%xmm0, %%xmm1 \n\t" + "movss %%xmm1, (%%rdx,%%rsi,1) \n\t" + "pshufd $0x39, %%xmm1, %%xmm2 \n\t" + "movss %%xmm2, (%%rdx,%%rsi,2) \n\t" + "pshufd $0x39, %%xmm2, %%xmm3 \n\t" + "movss %%xmm3, (%%rdx,%%r11 ) \n\t" + " \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "mulps %%xmm6, %%xmm11 \n\t" // scale by alpha, + "movaps %%xmm11, %%xmm0 \n\t" + " \n\t" + "movss %%xmm0, (%%rcx ) \n\t" // and store back to memory. + "pshufd $0x39, %%xmm0, %%xmm1 \n\t" + "movss %%xmm1, (%%rcx,%%rsi,1) \n\t" + "pshufd $0x39, %%xmm1, %%xmm2 \n\t" + "movss %%xmm2, (%%rcx,%%rsi,2) \n\t" + "pshufd $0x39, %%xmm2, %%xmm3 \n\t" + "movss %%xmm3, (%%rcx,%%r11 ) \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "mulps %%xmm6, %%xmm15 \n\t" // scale by alpha, + "movaps %%xmm15, %%xmm0 \n\t" + " \n\t" + "movss %%xmm0, (%%rdx ) \n\t" // and store back to memory. + "pshufd $0x39, %%xmm0, %%xmm1 \n\t" + "movss %%xmm1, (%%rdx,%%rsi,1) \n\t" + "pshufd $0x39, %%xmm1, %%xmm2 \n\t" + "movss %%xmm2, (%%rdx,%%rsi,2) \n\t" + "pshufd $0x39, %%xmm2, %%xmm3 \n\t" + "movss %%xmm3, (%%rdx,%%r11 ) \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "jmp .SDONE \n\t" // jump to end. " \n\t" " \n\t" " \n\t" @@ -770,16 +881,16 @@ void bli_dgemm_opt_4x4( " \n\t" "prefetcht2 3 * 8(%%rcx) \n\t" // prefetch c + 0*cs_c "xorpd %%xmm8, %%xmm8 \n\t" - "movaps %%xmm8, %%xmm9 \n\t" + "xorpd %%xmm9, %%xmm9 \n\t" "prefetcht2 3 * 8(%%rcx,%%rdi) \n\t" // prefetch c + 1*cs_c - "movaps %%xmm8, %%xmm10 \n\t" - "movaps %%xmm8, %%xmm11 \n\t" + "xorpd %%xmm10, %%xmm10 \n\t" + "xorpd %%xmm11, %%xmm11 \n\t" "prefetcht2 3 * 8(%%r10) \n\t" // prefetch c + 2*cs_c - "movaps %%xmm8, %%xmm12 \n\t" - "movaps %%xmm8, %%xmm13 \n\t" + "xorpd %%xmm12, %%xmm12 \n\t" + "xorpd %%xmm13, %%xmm13 \n\t" "prefetcht2 3 * 8(%%r10,%%rdi) \n\t" // prefetch c + 3*cs_c - "movaps %%xmm8, %%xmm14 \n\t" - "movaps %%xmm8, %%xmm15 \n\t" + "xorpd %%xmm14, %%xmm14 \n\t" + "xorpd %%xmm15, %%xmm15 \n\t" " \n\t" " \n\t" " \n\t" From 30e5eb29e060b97752f702d2ea5d101d950f53b2 Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Fri, 13 Nov 2015 12:14:19 -0600 Subject: [PATCH 36/38] Minor changes to treatment of rs, cs in bli_obj.c. Details: - Applied a patch submitted by Devin Matthews that: - implements subtle changes to handling of somewhat unusual cases of row and column strides to accommodate certail tensor cases, which includes adding dimension parameters to _is_col_tilted() and _is_row_tilted() macros, - simplifies how buffers are sized when requested BLIS-allocated objects, - re-consolidates bli_adjust_strides_*() into one function, and - defines 'restrict' keyword as a "nothing" macro for C++ and pre-C99 environments. --- frame/base/bli_check.c | 6 +- frame/base/bli_obj.c | 136 ++++++++------------------- frame/base/bli_obj.h | 16 ++-- frame/include/bli_macro_defs.h | 4 + frame/include/bli_param_macro_defs.h | 16 ++-- 5 files changed, 63 insertions(+), 115 deletions(-) diff --git a/frame/base/bli_check.c b/frame/base/bli_check.c index f2cfdf85ab..d333a40650 100644 --- a/frame/base/bli_check.c +++ b/frame/base/bli_check.c @@ -550,8 +550,10 @@ err_t bli_check_matrix_strides( dim_t m, dim_t n, inc_t rs, inc_t cs, inc_t is ) { if ( rs == 1 && cs == 1 ) { - // Only allow rs == cs == 1 for scalars. - if ( m > 1 || n > 1 ) + // If rs == cs == 1, then we must be dealing with an m-by-1, a + // 1-by-n, or a 1-by-1 matrix and thus at least one of the + // dimensions, m or n, must be unit (even if the other is zero). + if ( m != 1 && n != 1 ) return BLIS_INVALID_DIM_STRIDE_COMBINATION; } else if ( rs == 1 ) diff --git a/frame/base/bli_obj.c b/frame/base/bli_obj.c index 3395ec1b5e..d8d1520031 100644 --- a/frame/base/bli_obj.c +++ b/frame/base/bli_obj.c @@ -117,32 +117,24 @@ void bli_obj_alloc_buffer( inc_t rs, { dim_t n_elem = 0; dim_t m, n; - inc_t rs_abs, cs_abs, is_abs; siz_t elem_size; siz_t buffer_size; void* p; - m = bli_obj_length( *obj ); - n = bli_obj_width( *obj ); + // Query the dimensions of the object we are allocating. + m = bli_obj_length( *obj ); + n = bli_obj_width( *obj ); + + // Query the size of one element. + elem_size = bli_obj_elem_size( *obj ); // Adjust the strides, if needed, before doing anything else // (particularly, before doing any error checking). - bli_adjust_strides_alloc( m, n, &rs, &cs, &is ); + bli_adjust_strides( m, n, elem_size, &rs, &cs, &is ); if ( bli_error_checking_is_enabled() ) bli_obj_alloc_buffer_check( rs, cs, is, obj ); - // Query the size of one element. - elem_size = bli_obj_elem_size( *obj ); - - // Compute the magnitude of the row, column, and imaginary strides. - // We will use these in the comparisons below since those - // comparisions really relate only to the magnitudes of the strides, - // not their signs. - rs_abs = bli_abs( rs ); - cs_abs = bli_abs( cs ); - is_abs = bli_abs( is ); - // Determine how much object to allocate. if ( m == 0 || n == 0 ) { @@ -150,44 +142,13 @@ void bli_obj_alloc_buffer( inc_t rs, // should remain unchanged (because alignment is not needed). n_elem = 0; } - else if ( rs_abs == 1 ) - { - cs = bli_align_dim_to_size( cs, elem_size, - BLIS_HEAP_STRIDE_ALIGN_SIZE ); - n_elem = bli_abs( cs ) * n; - } - else if ( cs_abs == 1 ) - { - rs = bli_align_dim_to_size( rs, elem_size, - BLIS_HEAP_STRIDE_ALIGN_SIZE ); - n_elem = bli_abs( rs ) * m; - } else { - if ( rs_abs < cs_abs ) - { - // Note this case is identical to that of rs == 1 above. - cs = bli_align_dim_to_size( cs, elem_size, - BLIS_HEAP_STRIDE_ALIGN_SIZE ); - n_elem = bli_abs( cs ) * n; - } - else if ( cs_abs < rs_abs ) - { - // Note this case is identical to that of cs == 1 above. - rs = bli_align_dim_to_size( rs, elem_size, - BLIS_HEAP_STRIDE_ALIGN_SIZE ); - n_elem = bli_abs( rs ) * m; - } - else // if ( cs_abs == rs_abs ) - { - // If the row and column strides are equal, it almost certainly - // means that m == n == 1. (If that is not the case, then - // something is very wrong.) - if ( m != 1 || n != 1 ) - bli_check_error_code( BLIS_NOT_YET_IMPLEMENTED ); - - n_elem = 1; - } + // The number of elements to allocate is given by the distance from + // the element with the lowest address (usually {0, 0}) to the element + // with the highest address (usually {m-1, n-1}), plus one for the + // highest element itself. + n_elem = (m-1) * bli_abs( rs ) + (n-1) * bli_abs( cs ) + 1; } // Handle the special case where imaginary stride is larger than @@ -197,10 +158,7 @@ void bli_obj_alloc_buffer( inc_t rs, // Notice that adding is/2 works regardless of whether the // imaginary stride is unit, something between unit and // 2*n_elem, or something bigger than 2*n_elem. - if ( 1 > is_abs ) - { - n_elem = is_abs / 2 + n_elem; - } + n_elem = bli_abs( is ) / 2 + n_elem; } // Compute the size of the total buffer to be allocated, which includes @@ -222,16 +180,9 @@ void bli_obj_attach_buffer( void* p, inc_t is, obj_t* obj ) { - dim_t m, n; - - m = bli_obj_length( *obj ); - n = bli_obj_width( *obj ); - - // Adjust the strides, if necessary. - bli_adjust_strides_attach( m, n, &rs, &cs, &is ); - - // Notice that we wait until after strides have been adjusted to - // error-check. + // Check that the strides and lengths are compatible. Note that the + // user *must* specify valid row and column strides when attaching an + // external buffer. if ( bli_error_checking_is_enabled() ) bli_obj_attach_buffer_check( p, rs, cs, is, obj ); @@ -357,42 +308,20 @@ void bli_obj_create_const_copy_of( obj_t* a, obj_t* b ) *temp_i = ( gint_t ) bli_zreal( value ); } -void bli_adjust_strides_alloc( dim_t m, - dim_t n, - inc_t* rs, - inc_t* cs, - inc_t* is ) +void bli_adjust_strides( dim_t m, + dim_t n, + siz_t elem_size, + inc_t* rs, + inc_t* cs, + inc_t* is ) { // Here, we check the strides that were input from the user and modify // them if needed. // Handle the special "empty" case first. If either dimension is zero, - // we set row and column strides to zero. - if ( m == 0 || n == 0 ) - { - // NOTE: Previously, we forced the row and column strides to be zero - // if the matrices was being created with a zero dimension. It is - // *probably* the case that this is not necessary. However, in case - // a problem turns up later on, I'm leaving these lines commented - // out. - //*rs = 0; - //*cs = 0; - //*is = 1; - - return; - } - - bli_adjust_strides_attach( m, n, rs, cs, is ); -} - -void bli_adjust_strides_attach( dim_t m, - dim_t n, - inc_t* rs, - inc_t* cs, - inc_t* is ) -{ - // Here, we check the strides that were input from the user and modify - // them if needed. + // do nothing (this could represent a zero-length "slice" of another + // matrix). + if ( m == 0 || n == 0 ) return; // Interpret rs = cs = 0 as request for column storage. if ( *rs == 0 && *cs == 0 && ( *is == 0 || *is == 1 ) ) @@ -418,6 +347,21 @@ void bli_adjust_strides_attach( dim_t m, // Use default complex storage. *is = 1; + + // Align the strides depending on the tilt of the matrix. Note that + // scalars are neither row nor column tilted. Also note that alignment + // is only done for rs = cs = 0, and any user-supplied row and column + // strides are preserved. + if ( bli_is_col_tilted( m, n, *rs, *cs ) ) + { + *cs = bli_align_dim_to_size( *cs, elem_size, + BLIS_HEAP_STRIDE_ALIGN_SIZE ); + } + else if ( bli_is_row_tilted( m, n, *rs, *cs ) ) + { + *rs = bli_align_dim_to_size( *rs, elem_size, + BLIS_HEAP_STRIDE_ALIGN_SIZE ); + } } else if ( *rs == 1 && *cs == 1 ) { diff --git a/frame/base/bli_obj.h b/frame/base/bli_obj.h index d9926a52d8..92e9b1d874 100644 --- a/frame/base/bli_obj.h +++ b/frame/base/bli_obj.h @@ -78,16 +78,12 @@ void bli_obj_create_const( double value, obj_t* obj ); void bli_obj_create_const_copy_of( obj_t* a, obj_t* b ); -void bli_adjust_strides_alloc( dim_t m, - dim_t n, - inc_t* rs, - inc_t* cs, - inc_t* is ); -void bli_adjust_strides_attach( dim_t m, - dim_t n, - inc_t* rs, - inc_t* cs, - inc_t* is ); +void bli_adjust_strides( dim_t m, + dim_t n, + siz_t elem_size, + inc_t* rs, + inc_t* cs, + inc_t* is ); siz_t bli_datatype_size( num_t dt ); diff --git a/frame/include/bli_macro_defs.h b/frame/include/bli_macro_defs.h index 95c8580027..797c5d6118 100644 --- a/frame/include/bli_macro_defs.h +++ b/frame/include/bli_macro_defs.h @@ -40,12 +40,16 @@ #ifdef __cplusplus // Language is C++; define restrict as nothing. + #ifndef restrict #define restrict + #endif #elif __STDC_VERSION__ >= 199901L // Language is C99 (or later); do nothing since restrict is recognized. #else // Language is pre-C99; define restrict as nothing. + #ifndef restrict #define restrict + #endif #endif diff --git a/frame/include/bli_param_macro_defs.h b/frame/include/bli_param_macro_defs.h index c2e05ccaca..6be16bc0c0 100644 --- a/frame/include/bli_param_macro_defs.h +++ b/frame/include/bli_param_macro_defs.h @@ -378,13 +378,15 @@ ( bli_abs( rs ) != 1 && \ bli_abs( cs ) != 1 ) -#define bli_is_row_tilted( rs, cs ) \ +#define bli_is_row_tilted( m, n, rs, cs ) \ \ - ( bli_abs( cs ) < bli_abs( rs ) ) + ( bli_abs( cs ) == bli_abs( rs ) ? n < m \ + : bli_abs( cs ) < bli_abs( rs ) ) -#define bli_is_col_tilted( rs, cs ) \ +#define bli_is_col_tilted( m, n, rs, cs ) \ \ - ( bli_abs( rs ) < bli_abs( cs ) ) + ( bli_abs( rs ) == bli_abs( cs ) ? m < n \ + : bli_abs( rs ) < bli_abs( cs ) ) #define bli_has_nonunit_inc1( inc1 ) \ \ @@ -780,7 +782,7 @@ uplo_eff = uploa; \ diagoff_eff = diagoffa_use; \ \ - if ( bli_is_row_tilted( inca, lda ) ) \ + if ( bli_is_row_tilted( n_elem_max, n_iter_max, inca, lda ) ) \ { \ bli_swap_dims( n_iter_max, n_elem_max ); \ bli_swap_incs( inca, lda ); \ @@ -967,8 +969,8 @@ bli_negate_diag_offset( diagoff_eff ); \ } \ \ - if ( bli_is_row_tilted( incb, ldb ) && \ - bli_is_row_tilted( inca, lda ) ) \ + if ( bli_is_row_tilted( n_elem_max, n_iter_max, incb, ldb ) && \ + bli_is_row_tilted( n_elem_max, n_iter_max, inca, lda ) ) \ { \ bli_swap_dims( n_iter_max, n_elem_max ); \ bli_swap_incs( inca, lda ); \ From 0b126de1342c11c65623bcb38e258e21e9244e3d Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Fri, 13 Nov 2015 16:29:12 -0600 Subject: [PATCH 37/38] Consolidated packm_blk_var1 and packm_blk_var2. Details: - Consolidated the two blocked variants for packm into a single implementation (packm_blk_var1) and removed the other variant. - Updated all induced method _cntl_init() functions in frame/cntl/ind/ to use the new blocked variant 1. - Defined two new macros, bli_is_ind_packed() and bli_is_nat_packed(), to detect pack_t schemas for induced methods and native execution, respectively. --- frame/1m/packm/bli_packm.h | 1 - frame/1m/packm/bli_packm_blk_var1.c | 206 ++++++++++++++--- ...km_blk_var2.c => bli_packm_blk_var1.c.old} | 210 +++--------------- frame/1m/packm/bli_packm_blk_var2.h | 67 ------ frame/1m/packm/bli_packm_int.c | 2 +- frame/include/bli_param_macro_defs.h | 11 +- frame/ind/cntl/bli_gemm3m1_cntl.c | 4 +- frame/ind/cntl/bli_gemm3m2_cntl.c | 4 +- frame/ind/cntl/bli_gemm3m3_cntl.c | 2 +- frame/ind/cntl/bli_gemm3mh_cntl.c | 12 +- frame/ind/cntl/bli_gemm4m1_cntl.c | 4 +- frame/ind/cntl/bli_gemm4mb_cntl.c | 4 +- frame/ind/cntl/bli_gemm4mh_cntl.c | 8 +- frame/ind/cntl/bli_trsm3m1_cntl.c | 8 +- frame/ind/cntl/bli_trsm4m1_cntl.c | 8 +- 15 files changed, 242 insertions(+), 309 deletions(-) rename frame/1m/packm/{bli_packm_blk_var2.c => bli_packm_blk_var1.c.old} (67%) delete mode 100644 frame/1m/packm/bli_packm_blk_var2.h diff --git a/frame/1m/packm/bli_packm.h b/frame/1m/packm/bli_packm.h index c2d6b40766..26030bb2d9 100644 --- a/frame/1m/packm/bli_packm.h +++ b/frame/1m/packm/bli_packm.h @@ -42,7 +42,6 @@ #include "bli_packm_unb_var1.h" #include "bli_packm_blk_var1.h" -#include "bli_packm_blk_var2.h" #include "bli_packm_struc_cxk.h" #include "bli_packm_struc_cxk_4mi.h" diff --git a/frame/1m/packm/bli_packm_blk_var1.c b/frame/1m/packm/bli_packm_blk_var1.c index 78d52c9cad..3c0318bad8 100644 --- a/frame/1m/packm/bli_packm_blk_var1.c +++ b/frame/1m/packm/bli_packm_blk_var1.c @@ -62,6 +62,9 @@ typedef void (*FUNCPTR_T)( static FUNCPTR_T GENARRAY(ftypes,packm_blk_var1); extern func_t* packm_struc_cxk_kers; +extern func_t* packm_struc_cxk_4mi_kers; +extern func_t* packm_struc_cxk_3mis_kers; +extern func_t* packm_struc_cxk_rih_kers; void bli_packm_blk_var1( obj_t* c, @@ -96,6 +99,8 @@ void bli_packm_blk_var1( obj_t* c, dim_t pd_p = bli_obj_panel_dim( *p ); inc_t ps_p = bli_obj_panel_stride( *p ); + obj_t kappa; + obj_t* kappa_p; void* buf_kappa; func_t* packm_kers; @@ -103,14 +108,61 @@ void bli_packm_blk_var1( obj_t* c, FUNCPTR_T f; - // This variant assumes that the micro-kernel will always apply the - // alpha scalar of the higher-level operation. Thus, we use BLIS_ONE - // for kappa so that the underlying packm implementation does not - // scale during packing. - buf_kappa = bli_obj_buffer_for_const( dt_cp, BLIS_ONE ); + // Treatment of kappa (ie: packing during scaling) depends on + // whether we are executing an induced method. + if ( bli_is_ind_packed( schema ) ) + { + // The value for kappa we use will depend on whether the scalar + // attached to A has a nonzero imaginary component. If it does, + // then we will apply the scalar during packing to facilitate + // implementing induced complex domain algorithms in terms of + // real domain micro-kernels. (In the aforementioned situation, + // applying a real scalar is easy, but applying a complex one is + // harder, so we avoid the need altogether with the code below.) + if( thread_am_ochief( t ) ) + { + if ( bli_obj_scalar_has_nonzero_imag( p ) ) + { + // Detach the scalar. + bli_obj_scalar_detach( p, &kappa ); + + // Reset the attached scalar (to 1.0). + bli_obj_scalar_reset( p ); + + kappa_p = κ + } + else + { + // If the internal scalar of A has only a real component, then + // we will apply it later (in the micro-kernel), and so we will + // use BLIS_ONE to indicate no scaling during packing. + kappa_p = &BLIS_ONE; + } + } + kappa_p = thread_obroadcast( t, kappa_p ); + + // Acquire the buffer to the kappa chosen above. + buf_kappa = bli_obj_buffer_for_1x1( dt_cp, *kappa_p ); + } + else // if ( bli_is_nat_packed( schema ) ) + { + // This branch if for native execution, where we assume that + // the micro-kernel will always apply the alpha scalar of the + // higher-level operation. Thus, we use BLIS_ONE for kappa so + // that the underlying packm implementation does not perform + // any scaling during packing. + buf_kappa = bli_obj_buffer_for_const( dt_cp, BLIS_ONE ); + } - // Choose the correct func_t object. - packm_kers = packm_struc_cxk_kers; + + // Choose the correct func_t object based on the pack_t schema. + if ( bli_is_4mi_packed( schema ) ) packm_kers = packm_struc_cxk_4mi_kers; + else if ( bli_is_3mi_packed( schema ) || + bli_is_3ms_packed( schema ) ) packm_kers = packm_struc_cxk_3mis_kers; + else if ( bli_is_ro_packed( schema ) || + bli_is_io_packed( schema ) || + bli_is_rpi_packed( schema ) ) packm_kers = packm_struc_cxk_rih_kers; + else packm_kers = packm_struc_cxk_kers; // Query the datatype-specific function pointer from the func_t object. packm_ker = bli_func_obj_query( dt_cp, packm_kers ); @@ -144,8 +196,8 @@ void bli_packm_blk_var1( obj_t* c, } -#undef GENTFUNC -#define GENTFUNC( ctype, ch, varname, kertype ) \ +#undef GENTFUNCR +#define GENTFUNCR( ctype, ctype_r, ch, chr, varname, kertype ) \ \ void PASTEMAC(ch,varname)( \ struc_t strucc, \ @@ -204,6 +256,9 @@ void PASTEMAC(ch,varname)( \ conj_t conjc; \ bool_t row_stored; \ bool_t col_stored; \ + inc_t is_p_use; \ + dim_t ss_num; \ + dim_t ss_den; \ \ ctype* restrict c_use; \ ctype* restrict p_use; \ @@ -274,6 +329,17 @@ void PASTEMAC(ch,varname)( \ m_panel_max = &panel_dim_max; \ n_panel_max = &panel_len_max_i; \ } \ +\ + /* Compute the storage stride scaling. Usually this is just 1. However, + in the case of interleaved 3m, we need to scale by 3/2, and in the + cases of real-only, imag-only, or summed-only, we need to scale by + 1/2. In both cases, we are compensating for the fact that pointer + arithmetic occurs in terms of complex elements rather than real + elements. */ \ + if ( bli_is_3mi_packed( schema ) ) { ss_num = 3; ss_den = 2; } \ + else if ( bli_is_3ms_packed( schema ) ) { ss_num = 1; ss_den = 2; } \ + else if ( bli_is_rih_packed( schema ) ) { ss_num = 1; ss_den = 2; } \ + else { ss_num = 1; ss_den = 1; } \ \ /* Compute the total number of iterations we'll need. */ \ num_iter = iter_dim / panel_dim_max + ( iter_dim % panel_dim_max ? 1 : 0 ); \ @@ -297,6 +363,15 @@ void PASTEMAC(ch,varname)( \ } \ \ p_begin = p_cast; \ +\ +/* +if ( row_stored ) \ +PASTEMAC(ch,fprintm)( stdout, "packm_var2: b", m, n, \ + c_cast, rs_c, cs_c, "%4.1f", "" ); \ +if ( col_stored ) \ +PASTEMAC(ch,fprintm)( stdout, "packm_var2: a", m, n, \ + c_cast, rs_c, cs_c, "%4.1f", "" ); \ +*/ \ \ for ( ic = ic0, ip = ip0, it = 0; it < num_iter; \ ic += ic_inc, ip += ip_inc, it += 1 ) \ @@ -353,6 +428,15 @@ void PASTEMAC(ch,varname)( \ \ c_use = c_begin + (panel_off_i )*ldc; \ p_use = p_begin; \ +\ + /* We need to re-compute the imaginary stride as a function of + panel_len_max_i since triangular packed matrices have panels + of varying lengths. NOTE: This imaginary stride value is + only referenced by the packm kernels for induced methods. */ \ + is_p_use = ldp * panel_len_max_i; \ +\ + /* We nudge the imaginary stride up by one if it is odd. */ \ + is_p_use += ( bli_is_odd( is_p_use ) ? 1 : 0 ); \ \ if( packm_thread_my_iter( it, thread ) ) \ { \ @@ -370,25 +454,27 @@ void PASTEMAC(ch,varname)( \ kappa_cast, \ c_use, rs_c, cs_c, \ p_use, rs_p, cs_p, \ - is_p ); \ + is_p_use ); \ } \ \ /* NOTE: This value is usually LESS than ps_p because triangular matrices usually have several micro-panels that are shorter than a "full" micro-panel. */ \ - p_inc = ldp * panel_len_max_i; \ -\ - /* We nudge the panel increment up by one if it is odd. */ \ - p_inc += ( bli_is_odd( p_inc ) ? 1 : 0 ); \ + p_inc = ( is_p_use * ss_num ) / ss_den; \ } \ else if ( bli_is_herm_or_symm( strucc ) ) \ { \ /* This case executes if the panel belongs to a Hermitian or symmetric matrix, which includes stored, unstored, and diagonal-intersecting panels. */ \ +\ + c_use = c_begin; \ + p_use = p_begin; \ \ panel_len_i = panel_len_full; \ panel_len_max_i = panel_len_max; \ +\ + is_p_use = is_p; \ \ if( packm_thread_my_iter( it, thread ) ) \ { \ @@ -404,13 +490,11 @@ void PASTEMAC(ch,varname)( \ *m_panel_max, \ *n_panel_max, \ kappa_cast, \ - c_begin, rs_c, cs_c, \ - p_begin, rs_p, cs_p, \ - is_p ); \ + c_use, rs_c, cs_c, \ + p_use, rs_p, cs_p, \ + is_p_use ); \ } \ \ - /* NOTE: This value is equivalent to ps_p. */ \ - /*p_inc = ldp * panel_len_max_i;*/ \ p_inc = ps_p; \ } \ else \ @@ -418,9 +502,14 @@ void PASTEMAC(ch,varname)( \ /* This case executes if the panel is general, or, if the panel is part of a triangular matrix and is neither unstored (ie: zero) nor diagonal-intersecting. */ \ +\ + c_use = c_begin; \ + p_use = p_begin; \ \ panel_len_i = panel_len_full; \ panel_len_max_i = panel_len_max; \ +\ + is_p_use = is_p; \ \ if( packm_thread_my_iter( it, thread ) ) \ { \ @@ -436,28 +525,81 @@ void PASTEMAC(ch,varname)( \ *m_panel_max, \ *n_panel_max, \ kappa_cast, \ - c_begin, rs_c, cs_c, \ - p_begin, rs_p, cs_p, \ - is_p ); \ + c_use, rs_c, cs_c, \ + p_use, rs_p, cs_p, \ + is_p_use ); \ } \ -/* - if ( row_stored ) \ - PASTEMAC(ch,fprintm)( stdout, "packm_var1: bp copied", panel_len_max_i, panel_dim_max, \ - p_begin, rs_p, cs_p, "%9.2e", "" ); \ - else if ( col_stored ) \ - PASTEMAC(ch,fprintm)( stdout, "packm_var1: ap copied", panel_dim_max, panel_len_max_i, \ - p_begin, rs_p, cs_p, "%9.2e", "" ); \ -*/ \ \ /* NOTE: This value is equivalent to ps_p. */ \ - /*p_inc = ldp * panel_len_max_i;*/ \ p_inc = ps_p; \ } \ \ +/* + if ( bli_is_4mi_packed( schema ) ) { \ + printf( "packm_var2: is_p_use = %lu\n", is_p_use ); \ + if ( col_stored ) { \ + if ( 0 ) \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: a_r", *m_panel_use, *n_panel_use, \ + ( ctype_r* )c_use, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: ap_r", *m_panel_max, *n_panel_max, \ + ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: ap_i", *m_panel_max, *n_panel_max, \ + ( ctype_r* )p_use + is_p_use, rs_p, cs_p, "%4.1f", "" ); \ + } \ + if ( row_stored ) { \ + if ( 0 ) \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: b_r", *m_panel_use, *n_panel_use, \ + ( ctype_r* )c_use, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: bp_r", *m_panel_max, *n_panel_max, \ + ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: bp_i", *m_panel_max, *n_panel_max, \ + ( ctype_r* )p_use + is_p_use, rs_p, cs_p, "%4.1f", "" ); \ + } \ + } \ +*/ \ +/* +*/ \ +\ +/* +*/ \ +/* + PASTEMAC(chr,fprintm)( stdout, "packm_var2: bp_rpi", *m_panel_max, *n_panel_max, \ + ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ +*/ \ +\ +\ +/* + if ( row_stored ) { \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: b_r", *m_panel_max, *n_panel_max, \ + ( ctype_r* )c_use, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: b_i", *m_panel_max, *n_panel_max, \ + (( ctype_r* )c_use)+rs_c, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: bp_r", *m_panel_max, *n_panel_max, \ + ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ + inc_t is_b = rs_p * *m_panel_max; \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: bp_i", *m_panel_max, *n_panel_max, \ + ( ctype_r* )p_use + is_b, rs_p, cs_p, "%4.1f", "" ); \ + } \ +*/ \ +\ +\ +/* + if ( col_stored ) { \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: a_r", *m_panel_max, *n_panel_max, \ + ( ctype_r* )c_use, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: a_i", *m_panel_max, *n_panel_max, \ + (( ctype_r* )c_use)+rs_c, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: ap_r", *m_panel_max, *n_panel_max, \ + ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: ap_i", *m_panel_max, *n_panel_max, \ + ( ctype_r* )p_use + p_inc, rs_p, cs_p, "%4.1f", "" ); \ + } \ +*/ \ \ p_begin += p_inc; \ +\ } \ } -INSERT_GENTFUNC_BASIC( packm_blk_var1, packm_ker_t ) +INSERT_GENTFUNCR_BASIC( packm_blk_var1, packm_ker_t ) diff --git a/frame/1m/packm/bli_packm_blk_var2.c b/frame/1m/packm/bli_packm_blk_var1.c.old similarity index 67% rename from frame/1m/packm/bli_packm_blk_var2.c rename to frame/1m/packm/bli_packm_blk_var1.c.old index c53c8e6bcc..78d52c9cad 100644 --- a/frame/1m/packm/bli_packm_blk_var2.c +++ b/frame/1m/packm/bli_packm_blk_var1.c.old @@ -59,15 +59,12 @@ typedef void (*FUNCPTR_T)( packm_thrinfo_t* thread ); -//static FUNCPTR_T GENARRAY(ftypes,packm_blk_var2); +static FUNCPTR_T GENARRAY(ftypes,packm_blk_var1); extern func_t* packm_struc_cxk_kers; -extern func_t* packm_struc_cxk_4mi_kers; -extern func_t* packm_struc_cxk_3mis_kers; -extern func_t* packm_struc_cxk_rih_kers; -void bli_packm_blk_var2( obj_t* c, +void bli_packm_blk_var1( obj_t* c, obj_t* p, packm_thrinfo_t* t ) { @@ -99,8 +96,6 @@ void bli_packm_blk_var2( obj_t* c, dim_t pd_p = bli_obj_panel_dim( *p ); inc_t ps_p = bli_obj_panel_stride( *p ); - obj_t kappa; - obj_t* kappa_p; void* buf_kappa; func_t* packm_kers; @@ -108,59 +103,14 @@ void bli_packm_blk_var2( obj_t* c, FUNCPTR_T f; + // This variant assumes that the micro-kernel will always apply the + // alpha scalar of the higher-level operation. Thus, we use BLIS_ONE + // for kappa so that the underlying packm implementation does not + // scale during packing. + buf_kappa = bli_obj_buffer_for_const( dt_cp, BLIS_ONE ); -/* - // We want this variant to behave identically to that of variant 1 - // in the real domain. - if ( bli_is_real( dt_cp ) ) - { - bli_packm_blk_var1( c, p, t ); - return; - } -*/ - - // The value for kappa we use will depend on whether the scalar - // attached to A has a nonzero imaginary component. If it does, - // then we will apply the scalar during packing to facilitate - // implementing induced complex domain algorithms in terms of - // real domain micro-kernels. (In the aforementioned situation, - // applying a real scalar is easy, but applying a complex one is - // harder, so we avoid the need altogether with the code below.) - if( thread_am_ochief( t ) ) - { - if ( bli_obj_scalar_has_nonzero_imag( p ) ) - { - // Detach the scalar. - bli_obj_scalar_detach( p, &kappa ); - - // Reset the attached scalar (to 1.0). - bli_obj_scalar_reset( p ); - - kappa_p = κ - } - else - { - // If the internal scalar of A has only a real component, then - // we will apply it later (in the micro-kernel), and so we will - // use BLIS_ONE to indicate no scaling during packing. - kappa_p = &BLIS_ONE; - } - } - kappa_p = thread_obroadcast( t, kappa_p ); - - - // Acquire the buffer to the kappa chosen above. - buf_kappa = bli_obj_buffer_for_1x1( dt_cp, *kappa_p ); - - - // Choose the correct func_t object based on the pack_t schema. - if ( bli_is_4mi_packed( schema ) ) packm_kers = packm_struc_cxk_4mi_kers; - else if ( bli_is_3mi_packed( schema ) || - bli_is_3ms_packed( schema ) ) packm_kers = packm_struc_cxk_3mis_kers; - else if ( bli_is_ro_packed( schema ) || - bli_is_io_packed( schema ) || - bli_is_rpi_packed( schema ) ) packm_kers = packm_struc_cxk_rih_kers; - else packm_kers = packm_struc_cxk_kers; + // Choose the correct func_t object. + packm_kers = packm_struc_cxk_kers; // Query the datatype-specific function pointer from the func_t object. packm_ker = bli_func_obj_query( dt_cp, packm_kers ); @@ -168,9 +118,7 @@ void bli_packm_blk_var2( obj_t* c, // Index into the type combination array to extract the correct // function pointer. - //f = ftypes[dt_cp]; - if ( bli_is_scomplex( dt_cp ) ) f = bli_cpackm_blk_var2; - else f = bli_zpackm_blk_var2; + f = ftypes[dt_cp]; // Invoke the function. f( strucc, @@ -196,8 +144,8 @@ void bli_packm_blk_var2( obj_t* c, } -#undef GENTFUNCCO -#define GENTFUNCCO( ctype, ctype_r, ch, chr, varname, kertype ) \ +#undef GENTFUNC +#define GENTFUNC( ctype, ch, varname, kertype ) \ \ void PASTEMAC(ch,varname)( \ struc_t strucc, \ @@ -256,9 +204,6 @@ void PASTEMAC(ch,varname)( \ conj_t conjc; \ bool_t row_stored; \ bool_t col_stored; \ - inc_t is_p_use; \ - dim_t ss_num; \ - dim_t ss_den; \ \ ctype* restrict c_use; \ ctype* restrict p_use; \ @@ -329,17 +274,6 @@ void PASTEMAC(ch,varname)( \ m_panel_max = &panel_dim_max; \ n_panel_max = &panel_len_max_i; \ } \ -\ - /* Compute the storage stride scaling. Usually this is just 1. However, - in the case of interleaved 3m, we need to scale by 3/2, and in the - cases of real-only, imag-only, or summed-only, we need to scale by - 1/2. In both cases, we are compensating for the fact that pointer - arithmetic occurs in terms of complex elements rather than real - elements. */ \ - if ( bli_is_3mi_packed( schema ) ) { ss_num = 3; ss_den = 2; } \ - else if ( bli_is_3ms_packed( schema ) ) { ss_num = 1; ss_den = 2; } \ - else if ( bli_is_rih_packed( schema ) ) { ss_num = 1; ss_den = 2; } \ - else { ss_num = 1; ss_den = 1; } \ \ /* Compute the total number of iterations we'll need. */ \ num_iter = iter_dim / panel_dim_max + ( iter_dim % panel_dim_max ? 1 : 0 ); \ @@ -363,15 +297,6 @@ void PASTEMAC(ch,varname)( \ } \ \ p_begin = p_cast; \ -\ -/* -if ( row_stored ) \ -PASTEMAC(ch,fprintm)( stdout, "packm_var2: b", m, n, \ - c_cast, rs_c, cs_c, "%4.1f", "" ); \ -if ( col_stored ) \ -PASTEMAC(ch,fprintm)( stdout, "packm_var2: a", m, n, \ - c_cast, rs_c, cs_c, "%4.1f", "" ); \ -*/ \ \ for ( ic = ic0, ip = ip0, it = 0; it < num_iter; \ ic += ic_inc, ip += ip_inc, it += 1 ) \ @@ -428,14 +353,6 @@ PASTEMAC(ch,fprintm)( stdout, "packm_var2: a", m, n, \ \ c_use = c_begin + (panel_off_i )*ldc; \ p_use = p_begin; \ -\ - /* We need to re-compute the imaginary stride as a function of - panel_len_max_i since triangular packed matrices have panels - of varying lengths. */ \ - is_p_use = ldp * panel_len_max_i; \ -\ - /* We nudge the imaginary stride up by one if it is odd. */ \ - is_p_use += ( bli_is_odd( is_p_use ) ? 1 : 0 ); \ \ if( packm_thread_my_iter( it, thread ) ) \ { \ @@ -453,31 +370,25 @@ PASTEMAC(ch,fprintm)( stdout, "packm_var2: a", m, n, \ kappa_cast, \ c_use, rs_c, cs_c, \ p_use, rs_p, cs_p, \ - is_p_use ); \ + is_p ); \ } \ \ /* NOTE: This value is usually LESS than ps_p because triangular matrices usually have several micro-panels that are shorter than a "full" micro-panel. */ \ -/* p_inc = ldp * panel_len_max_i; \ +\ + /* We nudge the panel increment up by one if it is odd. */ \ p_inc += ( bli_is_odd( p_inc ) ? 1 : 0 ); \ -*/ \ - p_inc = ( is_p_use * ss_num ) / ss_den; \ } \ else if ( bli_is_herm_or_symm( strucc ) ) \ { \ /* This case executes if the panel belongs to a Hermitian or symmetric matrix, which includes stored, unstored, and diagonal-intersecting panels. */ \ -\ - c_use = c_begin; \ - p_use = p_begin; \ \ panel_len_i = panel_len_full; \ panel_len_max_i = panel_len_max; \ -\ - is_p_use = is_p; \ \ if( packm_thread_my_iter( it, thread ) ) \ { \ @@ -493,12 +404,13 @@ PASTEMAC(ch,fprintm)( stdout, "packm_var2: a", m, n, \ *m_panel_max, \ *n_panel_max, \ kappa_cast, \ - c_use, rs_c, cs_c, \ - p_use, rs_p, cs_p, \ - is_p_use ); \ + c_begin, rs_c, cs_c, \ + p_begin, rs_p, cs_p, \ + is_p ); \ } \ \ /* NOTE: This value is equivalent to ps_p. */ \ + /*p_inc = ldp * panel_len_max_i;*/ \ p_inc = ps_p; \ } \ else \ @@ -506,14 +418,9 @@ PASTEMAC(ch,fprintm)( stdout, "packm_var2: a", m, n, \ /* This case executes if the panel is general, or, if the panel is part of a triangular matrix and is neither unstored (ie: zero) nor diagonal-intersecting. */ \ -\ - c_use = c_begin; \ - p_use = p_begin; \ \ panel_len_i = panel_len_full; \ panel_len_max_i = panel_len_max; \ -\ - is_p_use = is_p; \ \ if( packm_thread_my_iter( it, thread ) ) \ { \ @@ -529,81 +436,28 @@ PASTEMAC(ch,fprintm)( stdout, "packm_var2: a", m, n, \ *m_panel_max, \ *n_panel_max, \ kappa_cast, \ - c_use, rs_c, cs_c, \ - p_use, rs_p, cs_p, \ - is_p_use ); \ + c_begin, rs_c, cs_c, \ + p_begin, rs_p, cs_p, \ + is_p ); \ } \ -\ - /* NOTE: This value is equivalent to ps_p. */ \ - p_inc = ps_p; \ - } \ -\ -/* - if ( bli_is_4mi_packed( schema ) ) { \ - printf( "packm_var2: is_p_use = %lu\n", is_p_use ); \ - if ( col_stored ) { \ - if ( 0 ) \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: a_r", *m_panel_use, *n_panel_use, \ - ( ctype_r* )c_use, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: ap_r", *m_panel_max, *n_panel_max, \ - ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: ap_i", *m_panel_max, *n_panel_max, \ - ( ctype_r* )p_use + is_p_use, rs_p, cs_p, "%4.1f", "" ); \ - } \ - if ( row_stored ) { \ - if ( 0 ) \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: b_r", *m_panel_use, *n_panel_use, \ - ( ctype_r* )c_use, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: bp_r", *m_panel_max, *n_panel_max, \ - ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: bp_i", *m_panel_max, *n_panel_max, \ - ( ctype_r* )p_use + is_p_use, rs_p, cs_p, "%4.1f", "" ); \ - } \ - } \ -*/ \ -/* -*/ \ -\ -/* -*/ \ /* - PASTEMAC(chr,fprintm)( stdout, "packm_var2: bp_rpi", *m_panel_max, *n_panel_max, \ - ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ + if ( row_stored ) \ + PASTEMAC(ch,fprintm)( stdout, "packm_var1: bp copied", panel_len_max_i, panel_dim_max, \ + p_begin, rs_p, cs_p, "%9.2e", "" ); \ + else if ( col_stored ) \ + PASTEMAC(ch,fprintm)( stdout, "packm_var1: ap copied", panel_dim_max, panel_len_max_i, \ + p_begin, rs_p, cs_p, "%9.2e", "" ); \ */ \ \ -\ -/* - if ( row_stored ) { \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: b_r", *m_panel_max, *n_panel_max, \ - ( ctype_r* )c_use, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: b_i", *m_panel_max, *n_panel_max, \ - (( ctype_r* )c_use)+rs_c, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: bp_r", *m_panel_max, *n_panel_max, \ - ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ - inc_t is_b = rs_p * *m_panel_max; \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: bp_i", *m_panel_max, *n_panel_max, \ - ( ctype_r* )p_use + is_b, rs_p, cs_p, "%4.1f", "" ); \ + /* NOTE: This value is equivalent to ps_p. */ \ + /*p_inc = ldp * panel_len_max_i;*/ \ + p_inc = ps_p; \ } \ -*/ \ -\ \ -/* - if ( col_stored ) { \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: a_r", *m_panel_max, *n_panel_max, \ - ( ctype_r* )c_use, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: a_i", *m_panel_max, *n_panel_max, \ - (( ctype_r* )c_use)+rs_c, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: ap_r", *m_panel_max, *n_panel_max, \ - ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: ap_i", *m_panel_max, *n_panel_max, \ - ( ctype_r* )p_use + p_inc, rs_p, cs_p, "%4.1f", "" ); \ - } \ -*/ \ \ p_begin += p_inc; \ -\ } \ } -INSERT_GENTFUNCCO_BASIC( packm_blk_var2, packm_ker_t ) +INSERT_GENTFUNC_BASIC( packm_blk_var1, packm_ker_t ) diff --git a/frame/1m/packm/bli_packm_blk_var2.h b/frame/1m/packm/bli_packm_blk_var2.h deleted file mode 100644 index 84e7b1cb5e..0000000000 --- a/frame/1m/packm/bli_packm_blk_var2.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - - BLIS - An object-based framework for developing high-performance BLAS-like - libraries. - - Copyright (C) 2014, The University of Texas at Austin - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - Neither the name of The University of Texas at Austin nor the names - of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -void bli_packm_blk_var2( obj_t* c, - obj_t* p, - packm_thrinfo_t* t ); - - -#undef GENTPROTCO -#define GENTPROTCO( ctype, ctype_r, ch, chr, varname ) \ -\ -void PASTEMAC(ch,varname)( \ - struc_t strucc, \ - doff_t diagoffc, \ - diag_t diagc, \ - uplo_t uploc, \ - trans_t transc, \ - pack_t schema, \ - bool_t invdiag, \ - bool_t revifup, \ - bool_t reviflo, \ - dim_t m, \ - dim_t n, \ - dim_t m_max, \ - dim_t n_max, \ - void* kappa, \ - void* c, inc_t rs_c, inc_t cs_c, \ - void* p, inc_t rs_p, inc_t cs_p, \ - inc_t is_p, \ - dim_t pd_p, inc_t ps_p, \ - void* packm_ker, \ - packm_thrinfo_t* t \ - ); - -INSERT_GENTPROTCO_BASIC( packm_blk_var2 ) - diff --git a/frame/1m/packm/bli_packm_int.c b/frame/1m/packm/bli_packm_int.c index 2a82871090..c90c0eaa57 100644 --- a/frame/1m/packm/bli_packm_int.c +++ b/frame/1m/packm/bli_packm_int.c @@ -44,7 +44,7 @@ static FUNCPTR_T vars[6][3] = { // unblocked optimized unblocked blocked { bli_packm_unb_var1, NULL, bli_packm_blk_var1 }, - { NULL, NULL, bli_packm_blk_var2 }, + { NULL, NULL, NULL, }, { NULL, NULL, NULL, }, { NULL, NULL, NULL, }, { NULL, NULL, NULL, }, diff --git a/frame/include/bli_param_macro_defs.h b/frame/include/bli_param_macro_defs.h index 6be16bc0c0..137fa4184a 100644 --- a/frame/include/bli_param_macro_defs.h +++ b/frame/include/bli_param_macro_defs.h @@ -654,6 +654,14 @@ bli_is_io_packed( schema ) || \ bli_is_rpi_packed( schema ) ) +#define bli_is_nat_packed( schema ) \ +\ + ( ( schema & BLIS_PACK_FORMAT_BITS ) == 0 ) + +#define bli_is_ind_packed( schema ) \ +\ + ( ( schema & BLIS_PACK_FORMAT_BITS ) != 0 ) + // pointer-related @@ -668,9 +676,6 @@ } - - - // return datatype for char #define bli_stype ( BLIS_FLOAT ) diff --git a/frame/ind/cntl/bli_gemm3m1_cntl.c b/frame/ind/cntl/bli_gemm3m1_cntl.c index ab9c98e7e2..909aedfd57 100644 --- a/frame/ind/cntl/bli_gemm3m1_cntl.c +++ b/frame/ind/cntl/bli_gemm3m1_cntl.c @@ -138,7 +138,7 @@ void bli_gemm3m1_cntl_init() gemm3m1_packa_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3m1_mr, gemm3m1_kr, FALSE, // do NOT invert diagonal @@ -150,7 +150,7 @@ void bli_gemm3m1_cntl_init() gemm3m1_packb_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3m1_kr, gemm3m1_nr, FALSE, // do NOT invert diagonal diff --git a/frame/ind/cntl/bli_gemm3m2_cntl.c b/frame/ind/cntl/bli_gemm3m2_cntl.c index 4b80bf69ff..7e362473ca 100644 --- a/frame/ind/cntl/bli_gemm3m2_cntl.c +++ b/frame/ind/cntl/bli_gemm3m2_cntl.c @@ -146,7 +146,7 @@ void bli_gemm3m2_cntl_init() gemm3m2_packa_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3m2_mr, gemm3m2_kr, FALSE, // do NOT invert diagonal @@ -158,7 +158,7 @@ void bli_gemm3m2_cntl_init() gemm3m2_packb_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3m2_kr, gemm3m2_nr, FALSE, // do NOT invert diagonal diff --git a/frame/ind/cntl/bli_gemm3m3_cntl.c b/frame/ind/cntl/bli_gemm3m3_cntl.c index f2193ca9fb..e444067fdf 100644 --- a/frame/ind/cntl/bli_gemm3m3_cntl.c +++ b/frame/ind/cntl/bli_gemm3m3_cntl.c @@ -144,7 +144,7 @@ void bli_gemm3m3_cntl_init() gemm3m3_packb_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3m3_kr, gemm3m3_nr, FALSE, // do NOT invert diagonal diff --git a/frame/ind/cntl/bli_gemm3mh_cntl.c b/frame/ind/cntl/bli_gemm3mh_cntl.c index 8e0b28a1d4..8f108fa16a 100644 --- a/frame/ind/cntl/bli_gemm3mh_cntl.c +++ b/frame/ind/cntl/bli_gemm3mh_cntl.c @@ -143,7 +143,7 @@ void bli_gemm3mh_cntl_init() gemm3mh_packa_cntl_ro = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3mh_mr, gemm3mh_kr, FALSE, // do NOT invert diagonal @@ -155,7 +155,7 @@ void bli_gemm3mh_cntl_init() gemm3mh_packb_cntl_ro = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3mh_kr, gemm3mh_nr, FALSE, // do NOT invert diagonal @@ -168,7 +168,7 @@ void bli_gemm3mh_cntl_init() gemm3mh_packa_cntl_io = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3mh_mr, gemm3mh_kr, FALSE, // do NOT invert diagonal @@ -180,7 +180,7 @@ void bli_gemm3mh_cntl_init() gemm3mh_packb_cntl_io = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3mh_kr, gemm3mh_nr, FALSE, // do NOT invert diagonal @@ -193,7 +193,7 @@ void bli_gemm3mh_cntl_init() gemm3mh_packa_cntl_rpi = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3mh_mr, gemm3mh_kr, FALSE, // do NOT invert diagonal @@ -205,7 +205,7 @@ void bli_gemm3mh_cntl_init() gemm3mh_packb_cntl_rpi = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3mh_kr, gemm3mh_nr, FALSE, // do NOT invert diagonal diff --git a/frame/ind/cntl/bli_gemm4m1_cntl.c b/frame/ind/cntl/bli_gemm4m1_cntl.c index aa2bfa303f..3fb517d529 100644 --- a/frame/ind/cntl/bli_gemm4m1_cntl.c +++ b/frame/ind/cntl/bli_gemm4m1_cntl.c @@ -135,7 +135,7 @@ void bli_gemm4m1_cntl_init() gemm4m1_packa_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm4m1_mr, gemm4m1_kr, FALSE, // do NOT invert diagonal @@ -147,7 +147,7 @@ void bli_gemm4m1_cntl_init() gemm4m1_packb_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm4m1_kr, gemm4m1_nr, FALSE, // do NOT invert diagonal diff --git a/frame/ind/cntl/bli_gemm4mb_cntl.c b/frame/ind/cntl/bli_gemm4mb_cntl.c index 4dddc69e99..c59ed3a989 100644 --- a/frame/ind/cntl/bli_gemm4mb_cntl.c +++ b/frame/ind/cntl/bli_gemm4mb_cntl.c @@ -137,7 +137,7 @@ void bli_gemm4mb_cntl_init() gemm4mb_packa_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm4mb_mr, gemm4mb_kr, FALSE, // do NOT invert diagonal @@ -149,7 +149,7 @@ void bli_gemm4mb_cntl_init() gemm4mb_packb_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm4mb_kr, gemm4mb_nr, FALSE, // do NOT invert diagonal diff --git a/frame/ind/cntl/bli_gemm4mh_cntl.c b/frame/ind/cntl/bli_gemm4mh_cntl.c index 2ce643fca1..2deb4ee095 100644 --- a/frame/ind/cntl/bli_gemm4mh_cntl.c +++ b/frame/ind/cntl/bli_gemm4mh_cntl.c @@ -145,7 +145,7 @@ void bli_gemm4mh_cntl_init() gemm4mh_packa_cntl_ro = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm4mh_mr, gemm4mh_kr, FALSE, // do NOT invert diagonal @@ -157,7 +157,7 @@ void bli_gemm4mh_cntl_init() gemm4mh_packb_cntl_ro = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm4mh_kr, gemm4mh_nr, FALSE, // do NOT invert diagonal @@ -170,7 +170,7 @@ void bli_gemm4mh_cntl_init() gemm4mh_packa_cntl_io = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm4mh_mr, gemm4mh_kr, FALSE, // do NOT invert diagonal @@ -182,7 +182,7 @@ void bli_gemm4mh_cntl_init() gemm4mh_packb_cntl_io = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm4mh_kr, gemm4mh_nr, FALSE, // do NOT invert diagonal diff --git a/frame/ind/cntl/bli_trsm3m1_cntl.c b/frame/ind/cntl/bli_trsm3m1_cntl.c index 642d9317f4..5c88cd688e 100644 --- a/frame/ind/cntl/bli_trsm3m1_cntl.c +++ b/frame/ind/cntl/bli_trsm3m1_cntl.c @@ -112,7 +112,7 @@ void bli_trsm3m1_cntl_init() trsm3m1_l_packa_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, // IMPORTANT: n dim multiple must be mr to // support right and bottom-right edge cases gemm3m1_mr, @@ -126,7 +126,7 @@ void bli_trsm3m1_cntl_init() trsm3m1_l_packb_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, // IMPORTANT: m dim multiple must be mr since // B_pack is updated (ie: serves as C) in trsm gemm3m1_mr, @@ -141,7 +141,7 @@ void bli_trsm3m1_cntl_init() trsm3m1_r_packa_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3m1_nr, gemm3m1_mr, FALSE, // do NOT invert diagonal @@ -153,7 +153,7 @@ void bli_trsm3m1_cntl_init() trsm3m1_r_packb_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3m1_mr, gemm3m1_mr, TRUE, // invert diagonal diff --git a/frame/ind/cntl/bli_trsm4m1_cntl.c b/frame/ind/cntl/bli_trsm4m1_cntl.c index 191e46f610..54883c42f2 100644 --- a/frame/ind/cntl/bli_trsm4m1_cntl.c +++ b/frame/ind/cntl/bli_trsm4m1_cntl.c @@ -112,7 +112,7 @@ void bli_trsm4m1_cntl_init() trsm4m1_l_packa_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, // IMPORTANT: n dim multiple must be mr to // support right and bottom-right edge cases gemm4m1_mr, @@ -126,7 +126,7 @@ void bli_trsm4m1_cntl_init() trsm4m1_l_packb_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, // IMPORTANT: m dim multiple must be mr since // B_pack is updated (ie: serves as C) in trsm gemm4m1_mr, @@ -141,7 +141,7 @@ void bli_trsm4m1_cntl_init() trsm4m1_r_packa_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm4m1_nr, gemm4m1_mr, FALSE, // do NOT invert diagonal @@ -153,7 +153,7 @@ void bli_trsm4m1_cntl_init() trsm4m1_r_packb_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm4m1_mr, gemm4m1_mr, TRUE, // invert diagonal From 3b358fa5bbe1739587f78ad24b9d49bb305907af Mon Sep 17 00:00:00 2001 From: Zhang Xianyi Date: Sat, 12 Dec 2015 00:12:17 +0800 Subject: [PATCH 38/38] Relax the condition for dgemm AVX micro kernel column store branch. Use unaligned vmovups for accessing matrix C. --- kernels/x86_64/avx/3/bli_gemm_asm_d8x4.c | 76 ++++++++++++------------ 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/kernels/x86_64/avx/3/bli_gemm_asm_d8x4.c b/kernels/x86_64/avx/3/bli_gemm_asm_d8x4.c index c143a8a5a8..dd952a57ae 100644 --- a/kernels/x86_64/avx/3/bli_gemm_asm_d8x4.c +++ b/kernels/x86_64/avx/3/bli_gemm_asm_d8x4.c @@ -1382,12 +1382,12 @@ void bli_dgemm_asm_8x4( " \n\t" // ie: aligned, ldim aligned, and " \n\t" // column-stored " \n\t" - "cmpq $8, %%rsi \n\t" // set ZF if (8*rs_c) == 8. - "sete %%bl \n\t" // bl = ( ZF == 1 ? 1 : 0 ); - "testq $31, %%rcx \n\t" // set ZF if c & 32 is zero. - "setz %%bh \n\t" // bh = ( ZF == 0 ? 1 : 0 ); - "testq $31, %%rdi \n\t" // set ZF if (8*cs_c) & 32 is zero. - "setz %%al \n\t" // al = ( ZF == 0 ? 1 : 0 ); + // "cmpq $8, %%rsi \n\t" // set ZF if (8*rs_c) == 8. + // "sete %%bl \n\t" // bl = ( ZF == 1 ? 1 : 0 ); + // "testq $31, %%rcx \n\t" // set ZF if c & 32 is zero. + // "setz %%bh \n\t" // bh = ( ZF == 0 ? 1 : 0 ); + // "testq $31, %%rdi \n\t" // set ZF if (8*cs_c) & 32 is zero. + // "setz %%al \n\t" // al = ( ZF == 0 ? 1 : 0 ); " \n\t" // and(bl,bh) followed by " \n\t" // and(bh,al) will reveal result " \n\t" @@ -1399,9 +1399,10 @@ void bli_dgemm_asm_8x4( " \n\t" " \n\t" " \n\t" // check if aligned/column-stored - "andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. - "andb %%bh, %%al \n\t" // set ZF if bh & al == 1. - "jne .DCOLSTORED \n\t" // jump to column storage case + // "andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. + // "andb %%bh, %%al \n\t" // set ZF if bh & al == 1. + "cmpq $8, %%rsi \n\t" // set ZF if (8*rs_c) == 8. + "je .DCOLSTORED \n\t" // jump to column storage case " \n\t" " \n\t" " \n\t" @@ -1536,53 +1537,53 @@ void bli_dgemm_asm_8x4( ".DCOLSTORED: \n\t" " \n\t" // update c00:c33 " \n\t" - "vmovapd (%%rcx), %%ymm0 \n\t" // load c00:c30, + "vmovups (%%rcx), %%ymm0 \n\t" // load c00:c30, "vmulpd %%ymm2, %%ymm0, %%ymm0 \n\t" // scale by beta, "vaddpd %%ymm9, %%ymm0, %%ymm0 \n\t" // add the gemm result, - "vmovapd %%ymm0, (%%rcx) \n\t" // and store back to memory. + "vmovups %%ymm0, (%%rcx) \n\t" // and store back to memory. "addq %%rdi, %%rcx \n\t" // c += cs_c; " \n\t" - "vmovapd (%%rcx), %%ymm0 \n\t" // load c01:c31, + "vmovups (%%rcx), %%ymm0 \n\t" // load c01:c31, "vmulpd %%ymm2, %%ymm0, %%ymm0 \n\t" // scale by beta, "vaddpd %%ymm11, %%ymm0, %%ymm0 \n\t" // add the gemm result, - "vmovapd %%ymm0, (%%rcx) \n\t" // and store back to memory. + "vmovups %%ymm0, (%%rcx) \n\t" // and store back to memory. "addq %%rdi, %%rcx \n\t" // c += cs_c; " \n\t" - "vmovapd (%%rcx), %%ymm0 \n\t" // load c02:c32, + "vmovups (%%rcx), %%ymm0 \n\t" // load c02:c32, "vmulpd %%ymm2, %%ymm0, %%ymm0 \n\t" // scale by beta, "vaddpd %%ymm13, %%ymm0, %%ymm0 \n\t" // add the gemm result, - "vmovapd %%ymm0, (%%rcx) \n\t" // and store back to memory. + "vmovups %%ymm0, (%%rcx) \n\t" // and store back to memory. "addq %%rdi, %%rcx \n\t" // c += cs_c; " \n\t" - "vmovapd (%%rcx), %%ymm0 \n\t" // load c03:c33, + "vmovups (%%rcx), %%ymm0 \n\t" // load c03:c33, "vmulpd %%ymm2, %%ymm0, %%ymm0 \n\t" // scale by beta, "vaddpd %%ymm15, %%ymm0, %%ymm0 \n\t" // add the gemm result, - "vmovapd %%ymm0, (%%rcx) \n\t" // and store back to memory. + "vmovups %%ymm0, (%%rcx) \n\t" // and store back to memory. " \n\t" " \n\t" // update c40:c73 " \n\t" - "vmovapd (%%rdx), %%ymm0 \n\t" // load c40:c70, + "vmovups (%%rdx), %%ymm0 \n\t" // load c40:c70, "vmulpd %%ymm2, %%ymm0, %%ymm0 \n\t" // scale by beta, "vaddpd %%ymm8, %%ymm0, %%ymm0 \n\t" // add the gemm result, - "vmovapd %%ymm0, (%%rdx) \n\t" // and store back to memory. + "vmovups %%ymm0, (%%rdx) \n\t" // and store back to memory. "addq %%rdi, %%rdx \n\t" // c += cs_c; " \n\t" - "vmovapd (%%rdx), %%ymm0 \n\t" // load c41:c71, + "vmovups (%%rdx), %%ymm0 \n\t" // load c41:c71, "vmulpd %%ymm2, %%ymm0, %%ymm0 \n\t" // scale by beta, "vaddpd %%ymm10, %%ymm0, %%ymm0 \n\t" // add the gemm result, - "vmovapd %%ymm0, (%%rdx) \n\t" // and store back to memory. + "vmovups %%ymm0, (%%rdx) \n\t" // and store back to memory. "addq %%rdi, %%rdx \n\t" // c += cs_c; " \n\t" - "vmovapd (%%rdx), %%ymm0 \n\t" // load c42:c72, + "vmovups (%%rdx), %%ymm0 \n\t" // load c42:c72, "vmulpd %%ymm2, %%ymm0, %%ymm0 \n\t" // scale by beta, "vaddpd %%ymm12, %%ymm0, %%ymm0 \n\t" // add the gemm result, - "vmovapd %%ymm0, (%%rdx) \n\t" // and store back to memory. + "vmovups %%ymm0, (%%rdx) \n\t" // and store back to memory. "addq %%rdi, %%rdx \n\t" // c += cs_c; " \n\t" - "vmovapd (%%rdx), %%ymm0 \n\t" // load c43:c73, + "vmovups (%%rdx), %%ymm0 \n\t" // load c43:c73, "vmulpd %%ymm2, %%ymm0, %%ymm0 \n\t" // scale by beta, "vaddpd %%ymm14, %%ymm0, %%ymm0 \n\t" // add the gemm result, - "vmovapd %%ymm0, (%%rdx) \n\t" // and store back to memory. + "vmovups %%ymm0, (%%rdx) \n\t" // and store back to memory. " \n\t" " \n\t" "jmp .DDONE \n\t" // jump to end. @@ -1591,10 +1592,11 @@ void bli_dgemm_asm_8x4( " \n\t" " \n\t" ".DBETAZERO: \n\t" - " \n\t" // check if aligned/column-stored - "andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. - "andb %%bh, %%al \n\t" // set ZF if bh & al == 1. - "jne .DCOLSTORBZ \n\t" // jump to column storage case + " \n\t" // check if column-stored + // "andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. + // "andb %%bh, %%al \n\t" // set ZF if bh & al == 1. + "cmpq $8, %%rsi \n\t" // set ZF if (8*rs_c) == 8. + "je .DCOLSTORBZ \n\t" // jump to column storage case " \n\t" " \n\t" " \n\t" @@ -1665,29 +1667,29 @@ void bli_dgemm_asm_8x4( ".DCOLSTORBZ: \n\t" " \n\t" // update c00:c33 " \n\t" - "vmovapd %%ymm9, (%%rcx) \n\t" // store c00:c30 + "vmovups %%ymm9, (%%rcx) \n\t" // store c00:c30 "addq %%rdi, %%rcx \n\t" // c += cs_c; " \n\t" - "vmovapd %%ymm11, (%%rcx) \n\t" // store c01:c31 + "vmovups %%ymm11, (%%rcx) \n\t" // store c01:c31 "addq %%rdi, %%rcx \n\t" // c += cs_c; " \n\t" - "vmovapd %%ymm13, (%%rcx) \n\t" // store c02:c32 + "vmovups %%ymm13, (%%rcx) \n\t" // store c02:c32 "addq %%rdi, %%rcx \n\t" // c += cs_c; " \n\t" - "vmovapd %%ymm15, (%%rcx) \n\t" // store c03:c33 + "vmovups %%ymm15, (%%rcx) \n\t" // store c03:c33 " \n\t" " \n\t" // update c40:c73 " \n\t" - "vmovapd %%ymm8, (%%rdx) \n\t" // store c40:c70 + "vmovups %%ymm8, (%%rdx) \n\t" // store c40:c70 "addq %%rdi, %%rdx \n\t" // c += cs_c; " \n\t" - "vmovapd %%ymm10, (%%rdx) \n\t" // store c41:c71 + "vmovups %%ymm10, (%%rdx) \n\t" // store c41:c71 "addq %%rdi, %%rdx \n\t" // c += cs_c; " \n\t" - "vmovapd %%ymm12, (%%rdx) \n\t" // store c42:c72 + "vmovups %%ymm12, (%%rdx) \n\t" // store c42:c72 "addq %%rdi, %%rdx \n\t" // c += cs_c; " \n\t" - "vmovapd %%ymm14, (%%rdx) \n\t" // store c43:c73 + "vmovups %%ymm14, (%%rdx) \n\t" // store c43:c73 " \n\t" " \n\t" " \n\t"