diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000..bfe7412a58 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,34 @@ +language: c +sudo: false + +compiler: + - gcc + +env: + - RUN_TEST=1 BUILD_CONFIG="auto" + - RUN_TEST=1 BUILD_CONFIG="reference" + - RUN_TEST=1 BUILD_CONFIG="dunnington" + - RUN_TEST=0 BUILD_CONFIG="sandybridge" + - RUN_TEST=0 BUILD_CONFIG="haswell" + - RUN_TEST=0 BUILD_CONFIG="bulldozer" + - RUN_TEST=0 BUILD_CONFIG="piledriver" + - RUN_TEST=0 BUILD_CONFIG="carrizo" + +install: +- if [ "$CC" = "gcc" ]; then export CXX="g++-4.8" CC="gcc-4.8"; fi +addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - gcc-4.8 + - g++-4.8 + - clang + + + +script: + - ./configure $BUILD_CONFIG + - make CC=gcc-4.8 + - if [ $RUN_TEST -eq 1 ]; then make BLIS_ENABLE_TEST_OUTPUT=yes test; fi + - if [ $RUN_TEST -eq 1 ]; then ./build/check-test.sh ./output.testsuite; fi \ No newline at end of file diff --git a/CHANGELOG b/CHANGELOG index ee843471e0..1c6b42d14f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,10 +1,873 @@ -commit 38ea5022e4ed846112198c4e1672fcdaeb90dc71 (HEAD, 0.1.6, master) +commit 47caa33485b91ea6f2a5e386e61210c90c5f489f (HEAD -> master, tag: 0.1.8) +Author: Field G. Van Zee +Date: Wed Jul 29 13:31:09 2015 -0500 + + Version file update (0.1.8) + +commit ef0fbbbdb6148b96938733fce72cb4ed7dad685e (origin/master) +Merge: fdfe14f d4b8913 +Author: Field G. Van Zee +Date: Thu Jul 9 13:54:54 2015 -0500 + + Merge branch 'master' of github.com:flame/blis + +commit fdfe14f1e17ba5a2f8dfa0bdb799c6b0e730211b +Author: Field G. Van Zee +Date: Thu Jul 9 13:52:39 2015 -0500 + + Added support for Intel Haswell/Broadwell. + + Details: + - Added sgemm and dgemm micro-kernels, which employ 256-bit AVX vectors + and FMA instructions. (Complex support is currently provided by default + induced method, 4m1a.) + - Added a 'haswell' configuration, which uses the aforementioned kernels. + - Inserted auto-detection support for haswell configuration in + build/auto-detect/cpuid_x86.c. + - Modified configure script to explicitly echo when automatic or manual + configuration is in progress. + - Changed beta scalar in test_gemm.c module of test suite to -1.0 to 0.9. + +commit d4b891369c1eb0879ade662ff896a5b9a7fca207 +Author: Field G. Van Zee +Date: Tue Jul 7 10:06:53 2015 -0500 + + Added 'carrizo' configuration. + + Details: + - Added a new configuration for AMD Excavator-based hardware also known + as Carrizo when referring to the entire APU. This configuration uses + the same micro-kernels as the piledriver, but with different + cache blocksizes. + +commit 0b7255a642d56723f02d7ca1f8f21809967b8515 +Author: Field G. Van Zee +Date: Fri Jun 19 12:01:50 2015 -0500 + + CHANGELOG update (0.1.7) + +commit 267253de8a7be546ce87626443ee38701c1d411f (tag: 0.1.7) +Author: Field G. Van Zee +Date: Fri Jun 19 12:01:49 2015 -0500 + + Version file update (0.1.7) + +commit 7cd01b71b5e757a6774625b3c9f427f5e7664a76 +Author: Field G. Van Zee +Date: Fri Jun 19 11:31:53 2015 -0500 + + Implemented dynamic allocation for packing buffers. + + Details: + - Replaced the old memory allocator, which was based on statically- + allocated arrays, with one based on a new internal pool_t type, which, + combined with a new bli_pool_*() API, provides a new abstract data + type that implements the same memory pool functionality but with blocks + from the heap (ie: malloc() or equivalent). Hiding the details of the + pool in a separate API also allows for a much simpler bli_mem.c family + of functions. + - Added a new internal header, bli_config_macro_defs.h, which enables + sane defaults for the values previously found in bli_config. Those + values can be overridden by #defining them in bli_config.h the same + way kernel defaults can be overridden in bli_kernel.h. This file most + resembles what was previously a typical configuration's bli_config.h. + - Added a new configuration macro, BLIS_POOL_ADDR_ALIGN_SIZE, which + defaults to BLIS_PAGE_SIZE, to specify the alignment of individual + blocks in the memory pool. Also added a corresponding query routine to + the bli_info API. + - Deprecated (once again) the micro-panel alignment feature. Upon further + reflection, it seems that the goal of more predictable L1 cache + replacement behavior is outweighed by the harm caused by non-contiguous + micro-panels when k % kc != 0. I honestly don't think anyone will even + miss this feature. + - Changed bli_ukr_get_funcs() and bli_ukr_get_ref_funcs() to call + bli_cntl_init() instead of bli_init(). + - Removed query functions from bli_info.c that are no longer applicable + given the dynamic memory allocator. + - Removed unnecessary definitions from configurations' bli_config.h files, + which are now pleasantly sparse. + - Fixed incorrect flop counts in addv, subv, scal2v, scal2m testsuite + modules. Thanks to Devangi Parikh for pointing out these + miscalculations. + - Comment, whitespace changes. + +commit 9848f255a3bab17d1139c391cca13ff3f1ffe6ed +Author: Field G. Van Zee +Date: Thu Jun 11 19:14:22 2015 -0500 + + Added early return to API-level _init() routines. + + Details: + - Added conditional code that returns early from the API-level _init() + routines if the API is already initialized. Actually meant for this to + be included in 5f93cbe8. + +commit 5f93cbe870f3478870e15581e7fd450dad5bba1e +Author: Field G. Van Zee +Date: Thu Jun 11 18:52:12 2015 -0500 + + Introduced API-level initialization. + + Details: + - Added API-level initialization state to _const, _error, _mem, _thread, + _ind, and _cntl APIs. While this functionality will mostly go unused, + adding miniscule overhead at init-time, there will be at least once + instance in the near future where, in order to avoid an infinite loop, + a certain portion of the initialization will call a query function that + itself attempts to call bli_init(). API-level initialization will allow + this later stage to verify that an earlier stage of initialization has + completed, even if the overall call to bli_init() has not yet returned. + - Added _is_initialized() functions for each API, setting the underlying + bool_t during _init() and unsetting it during _finalize(). + - Comment, whitespace changes. + +commit ee129c6b028bc5ac88da7c74fde72c49803742ff +Author: Field G. Van Zee +Date: Wed Jun 10 12:53:28 2015 -0500 + + Fixed bugs in _get_range(), _get_range_weighted(). + + Details: + - Fixed some bugs that only manifested in multithreaded instances of + some (non-gemm) level-3 operations. The bugs were related to invalid + allocation of "edge" cases to thread subpartitions. (Here, we define + an "edge" case to be one where the dimension being partitioned for + parallelism is not a whole multiple of whatever register blocksize + is needed in that dimension.) In BLIS, we always require edge cases + to be part of the bottom, right, or bottom-right subpartitions. + (This is so that zero-padding only has to happen at the bottom, right, + or bottom-right edges of micro-panels.) The previous implementations + of bli_get_range() and _get_range_weighted() did not adhere to this + implicit policy and thus produced bad ranges for some combinations of + operation, parameter cases, problem sizes, and n-way parallelism. + - As part of the above fix, the functions bli_get_range() and + _get_range_weighted() have been renamed to use _l2r, _r2l, _t2b, + and _b2t suffixes, similar to the partitioning functions. This is + an easy way to make sure that the variants are calling the right + version of each function. The function signatures have also been + changed slightly. + - Comment/whitespace updates. + - Removed unnecessary '/' from macros in bli_obj_macro_defs.h. + +commit 9135dfd69d39f3bbd75034f479f27a78dbfebcce +Author: Field G. Van Zee +Date: Fri Jun 5 13:37:44 2015 -0500 + + Minor updates to test/3m4m files. + +commit d62ceece943b20537ec4dd99f25136b9ba2ae340 +Author: Field G. Van Zee +Date: Wed Jun 3 12:56:45 2015 -0500 + + Minor update to test/3m4m/runme.sh. + + Details: + - Removed some stale script code that should have been removed + during 590bb3b8c. + +commit b6ee82a3d421c9c4f1eb6848c7c6e37aa46de799 +Author: Field G. Van Zee +Date: Wed Jun 3 12:14:23 2015 -0500 + + Minor cleanup to bli_init() and friends. + + Details: + - Spun-off initialization of global scalar constants to bli_const_init() + and of threading stuff to bli_thread_init(). + - Added some missing _finalize() functions, even when there is nothing + to do. + +commit 1213f5cebabc1637ce9dd45c4bfa87bb93677c29 +Author: Field G. Van Zee +Date: Tue Jun 2 13:27:47 2015 -0500 + + POSIX thread bugfixes/edits to bli_init.c, _mem.c. + + Details: + - Fixed a sort-of bug in bli_init.c whereby the wrong pthread mutex + was used to lock access to initialization/finalization actions. + But everything worked out okay as long as bli_init() was called by + single-threaded code. + - Changed to static initialization for memory allocator mutex in + bli_mem.c, and moved mutex to that file (from bli_init.c). + - Fixed some type mismatches in bli_threading_pthreads.c that resulted + in compiler warnings. + - Fixed a small memory leak with allocated-but-never-freed (and unused) + pthread_attr_t objects. + - Whitespace changes to bli_init.c and bli_mem.c. + +commit 590bb3b8c5c0389159c5a9451b6c156c5f237e8a +Author: Field G. Van Zee +Date: Sun May 24 16:02:53 2015 -0500 + + Backed-out adjusted dim changes to test/3m4m. + + Details: + - Reverted most changes applied during commit ec25807b. + +commit ec25807b26da943868f0d0517c3720e50181b8f9 +Author: Field G. Van Zee +Date: Fri Apr 10 13:23:50 2015 -0500 + + Tweaks to test/3m4m to test with adjusted dims. + + Details: + - Updated test/3m4m driver files to build test drivers that allow + comparision of real "asm_blis" results to complex "asm_blis" results, + except with the latter's problem sizes adjusted so that problems are + generated with equal flop counts. + +commit 426b6488580a92bf071a62dc319a9c837ce39821 +Author: Field G. Van Zee +Date: Wed Apr 8 15:12:21 2015 -0500 + + Fixed a packing bug that manifested in trsm_r. + + Details: + - Fixed a bug that caused a memory leak in the contiguous memory + allocator. Because packm_init() was using simple aliasing when + a subpartition object was marked as zeros by bli_acquire_mpart_*(), + the "destination" pack object's mem_t entry was being overwritten + by the corresponding field of the "source" object (which was likely + NULL). This prevented the block from being released back to the + memory allocator. But this bug only manifested when changing the + location of packing B from outside the var1 loop to inside the + var3 loop, and only for trsm with triangular B (side = right). The + bug was fixed by changing the type of alias used in packm_init() + when handling zero partition cases. Specifically, we now use + bli_obj_alias_for_packing(), which does not clobber the destination + (pack) object's mem_t field. Thanks to Devangi Parikh for this bug + report. + +commit c84286d5cef48f16d83831baac1f46b9856b9a36 +Author: Field G. Van Zee +Date: Sat Apr 4 15:39:14 2015 -0500 + + More minor tweaks to test/3m4m. + + Details: + - Added a line of output that forces matlab to allocate the entire array + up-front. + - Re-enabled real domain benchmarks in runme.sh, which were temporarily + disabled. + +commit 309717c8ebf4ef1369f15cf41340e13c25b41573 +Author: Field G. Van Zee +Date: Fri Apr 3 19:28:49 2015 -0500 + + More tweaks to test/3m4m, configurations. + + Details: + - Fixed incorrect number of mc_x_kc memory blocks in + sandybridge/bli_config.h. + - Enabled OpenMP multithreding in piledriver/bli_config.h. + - More updates to test/3m4m driver files. + +commit 4baf3b9c69b2f648be9e46e07ccc9859dd675828 +Author: Field G. Van Zee +Date: Fri Apr 3 16:44:32 2015 -0500 + + Tweaked test/3m4m driver, including acml support. + + Details: + - Added ACML support to test/3m4m driver Makefile and runme.sh script. + +commit a32f7c49ca4ea869d2a6c66818780f4321743d67 +Merge: 349e075 4bfd1ce +Author: Field G. Van Zee +Date: Fri Apr 3 08:28:11 2015 -0500 + + Merge pull request #23 from xianyi/master + + Add auto-detecting CPU on configure stage. + +commit 349e075ad6a8e2a1211d94f36d24828c9d44b052 +Author: Field G. Van Zee +Date: Thu Apr 2 18:12:28 2015 -0500 + + Tweaks to sandybridge config, test/3m4m driver. + + Details: + - Enable OpenMP support by default in sandybridge's bli_config.h. + - Reorganized sandybridge's bli_kernel.h. + - Updated 3m4m Makefile, runme.sh to also test MKL implementation. + +commit 4bfd1ce8ca93f93d170dd2715f0a32027b417b46 +Author: Zhang Xianyi +Date: Thu Apr 2 16:40:21 2015 -0500 + + Detect NEON for cortex-a9 and cortex-a15. + +commit aa6eec4f43137057276fe6119bdbfb5c52682527 +Author: Zhang Xianyi +Date: Thu Apr 2 16:03:44 2015 -0500 + + Detect the CPU architecture. Support ARM cores. + + Detect the CPU architecture by compiler's predefined macros. + Then, detect the CPU cores. + + Support detecting x86 and ARM architectures. + +commit 2947cfb749c937b0f62fac36cc92f123bd45b53c +Author: Zhang Xianyi +Date: Wed Apr 1 12:24:00 2015 -0500 + + Add auto-detecting CPU on configure stage. + e.g. /Path_to_BLIS/configure auto + + Now, it only support detecting x86 CPUs. + +commit 26a4b8f6f985597f80e0174990bf541f1d9bafac +Author: Field G. Van Zee +Date: Wed Apr 1 10:44:54 2015 -0500 + + Implemented 3m2, 3m3 induced algorithms (gemm only). + + Details: + - Defined a new "3ms" (separated 3m) pack schema and added appropriate + support in packm_init(), packm_blk_var2(). + - Generalized packm_struc_cxk_3mi to take the imaginary stride (is_p) + as an argument instead of computing it locally. Exception: for trmm, + is_p must be computed locally, since it changes for triangular + packed matrices. Also exposed is_p in interface to dt-specific + packm_blk_var2 (and _var1, even though it does not use imaginary + stride). + - Renamed many functions/variables from _3mi to _3mis to indicate that + they work for either interleaved or separated 3m pack schemas. + - Generalized gemm and herk macro-kernels to pass in imaginary stride + rather than compute them locally. + - Added support for 3m2 and 3m3 algorithms to frame/ind, including 3m2- + and 3m3-specific virtual micro-kernels. + - Added special gemm macro-kernels to support 3m2 and 3m3. + - Added support for 3m2 and 3m3 to testsuite. + - Corrected the type of the panel dimension (pd_) in various macro- + kernels from inc_t to dim_t. + - Renamed many functions defined in bli_blocksize.c. + - Moved most induced-related macro defs from frame/include to + frame/ind/include. + - Updated the _ukernel.c files so that the micro-kernel function pointers + are obtained from the func_t objects rather than the cpp macros that + define the function names. + - Updated test/3m4m driver, Makefile, and run script. + +commit ddf62ba7d2da08225b201585b85e06c967767dea +Author: Tyler Smith +Date: Fri Mar 27 14:27:51 2015 -0500 + + Refuse to free the packm thread info if it uses the single threaded version + +commit 016fc587584d958a0e430a56a5e2c05022ac2f17 +Author: Tyler Smith +Date: Fri Mar 27 14:23:02 2015 -0500 + + Don't free packm thread info if it is null + +commit 00a443c529a60862a57b93e303a0b3212c9b1df4 +Author: Tyler Smith +Date: Fri Mar 27 14:11:07 2015 -0500 + + Use bli_malloc instead of malloc for the thread info paths + +commit f1a6b7d02861ccebdc500ea98778cc0f6cddad17 +Author: Field G. Van Zee +Date: Wed Mar 18 15:37:10 2015 -0500 + + Reorganized code for induced complex methods. + + Details: + - Consolidated most of the code relating to induced complex methods + (e.g. 4mh, 4m1, 3mh, 3m1, etc.) into frame/ind. Induced methods + are now enabled on a per-operation basis. The current "available" + (enabled and implemented) implementation can then be queried on + an operation basis. Micro-kernel func_t objects as well as blksz_t + objects can also be queried in a similar maner. + - Redefined several micro-kernel and operation-related functions in + bli_info_*() API, in accordance with above changes. + - Added mr and nr fields to blksz_t object, which point to the mr + and nr blksz_t objects for each cache blocksize (and are NULL for + register blocksizes). Renamed the sub-blocksize field "sub" to + "mult" since it is really expressing a blocksize multiple. + - Updated bli_*_determine_kc_[fb]() for gemm/hemm/symm, trmm, and + trsm to correctly query mr and nr (for purposes of nudging kc). + - Introduced an enumerated opid_t in bli_type_defs.h that uniquely + identifies an operation. For now, only level-3 id values are defined, + along with a generic, catch-all BLIS_NOID value. + - Reworked testsuite so that all induced methods that are enabled + are tested (one at a time) rather than only testing the first + available method. + - Reformated summary at the beginning of testsuite output so that + blocksize and micro-kernel info is shown for each induced method + that was requested (as well as native execution). + - Reduced the number of columns needed to display non-matlab + testsuite output (from approx. 90 to 80). + +commit 8d5169ccda954e5f72944308a036dcb7ebfc9097 +Author: Field G. Van Zee +Date: Wed Mar 18 11:38:08 2015 -0500 + + Fixed bug in release of mem_t buffer. + + Details: + - Fixed a bug that affects all level-2 and level-3 blocked variants. The + bug only manifested, however, if the packing of operands (A and B in + gemm, for example) spanned multiple nodes in the control tree. Until + recently, the main consumers of packm were level-3 operations, all of + which packed both input operands from blocked variant 1 (B outside of + the loop, and A within the loop). This particular usage masked a flaw + in the code whereby bli_obj_release_pack() would always release the + underlying mem_t buffer (provided it was allocated), even if the buffer + was not allocated in the current variant. This has been fixed by + replacing all calls to bli_obj_release_pack() with calls to a new + function, bli_packm_release(), which takes the same control tree node + argument passed into the object's corresponding call to packm_init() + or packv_init(). bli_packm_release() then proceeds to invoke + bli_obj_release_pack() only if the control tree node indicates that + packing was requested. Thanks to Devangi Parikh for identifying this + bug. + +commit c0acca0f5182ba96fd39c9d10b34a896a6e74206 +Author: Field G. Van Zee +Date: Tue Mar 3 10:56:22 2015 -0600 + + Clarified comments in testsuite input.operations. + +commit 03ba9a6b17861d9e1adc0cf924439c4d7e860d19 +Author: Field G. Van Zee +Date: Tue Feb 24 10:33:28 2015 -0600 + + Removed some 'old' directories. + +commit a86db60ee270cdeb745ae7cf68f9e0becc9f522d +Author: Field G. Van Zee +Date: Mon Feb 23 18:42:39 2015 -0600 + + Extensive renaming of 3m/4m-related files, symbols. + + Details: + - Renamed all remaining 3m/4m packing files and symbols to 3mi/4mi + ('i' for "interleaved"). Similar changes to 3M/4M macros. + - Renamed all 3m/4m files and functions to 3m1/4m1. + - Whitespace changes. + +commit 8cf8da291a0fb2f491f410969a76ec0fbda47faf +Author: Field G. Van Zee +Date: Fri Feb 20 15:24:27 2015 -0600 + + Minor updates to induced complex mode management. + + Details: + - Relocated bli_4mh.c, bli_4mb.c, bli_4m.c, bli_3mh.c, bli_3m.c (and + associated headers) from frame/base to frame/base/induced. + - Added bli_xm.? to frame/base/induced, which implements + bli_xm_is_enabled(), which detects whether ANY induced complex method + is currently enabled. + - The new function bli_xm_is_enabled() is now used in bli_info.c to + detect when an induced complex method is used, so we know when to + return blocksizes from one of the induced methods' blocksize objects. + +commit 411e637ee7d1083a84f58f08938d51e63d7c3c9a +Merge: c2569b8 fc0b771 +Author: Tyler Michael Smith +Date: Fri Feb 20 20:39:25 2015 -0600 + + Merge branch 'master' of http://github.com/flame/blis + +commit c2569b8803d4ccc1d7b6f391713461b51443601d +Author: Tyler Michael Smith +Date: Fri Feb 20 20:38:19 2015 -0600 + + Fixed a memory leak in freeing the thread infos + +commit fc0b771227abf86d81f505b324f69f6e83db1d8f +Author: Field G. Van Zee +Date: Fri Feb 20 11:47:44 2015 -0600 + + Added max(mr,nr) to kc in static mem pools. + + Details: + - Changed the static memory definitions to compute the maximum register + blocksize for each datatype and add it to kc when computing the size + of blocks of A and B. This formally accounts for the nudging of kc + up to a multiple of mr or nr at runtime for triangular operations + (e.g. trmm). + +commit af32e3a608631953ef770341df10a14a991bf290 +Author: Tyler Michael Smith +Date: Thu Feb 19 22:51:11 2015 -0600 + + Fixed a bug with get_range_weighted would return end = 0 for small problem sizes + +commit 441d47542a64e131578d00da7404c1ed387a721c +Author: Field G. Van Zee +Date: Thu Feb 19 17:06:10 2015 -0600 + + Renamed 3m and 4m symbols/macros to 3mi and 4mi. + + Details: + - Renamed several variables and macros from 3m/4m to 3mi/4mi. This is + because those packing schemas were always implicitly "interleaved". + This new naming scheme will make way for new schemas that separate + instead of interleve the real and imaginary (and summed) parts. + - Expanded the pack format sub-field of the pack schema field of the + info_t to 4 bits (from 3). This will allow for more schema types + going forward. + - Removed old _cntl.c files for herk3m, herk4m, trmm3m, trmm4m. + +commit 518a1756ccf02122b96fc437b538604a597df42a +Author: Field G. Van Zee +Date: Thu Feb 19 14:27:09 2015 -0600 + + Fixed indexing bug for trmm3 via 3mh, 4mh. + + Details: + - Fixed a bug that only affected trmm3 when performed via 3mh or 4mh, + whereby micro-panels of the triangular matrix were packed with "dead + space" between them due to failing to adjust for the fact that pointer + arithmetic was occurring in units of complex elements while the data + being packed consisted of real elements. It turns out that the macro- + kernel suffered from the same bug, meaning the panels were actually + being packed and read consistently. The only way I was able to + discover the bug in the first place was because the packed block of A + was overflowing into the beginning of the packed row panel of B using + the sandybridge configuration. + +commit 493087d730f01d5169434f461644e5633f48a42f +Merge: 650d2a6 2502129 +Author: Field G. Van Zee +Date: Wed Feb 18 09:45:51 2015 -0600 + + Merge branch 'master' of github.com:flame/blis + +commit 25021299b670775df8ca9c87910c63d7e74ed946 +Merge: fe2b8d3 f05a576 +Author: Field G. Van Zee +Date: Wed Feb 11 20:03:21 2015 -0600 + + Merge branch 'master' of github.com:flame/blis + +commit fe2b8d39a445ac848686e78c7540fd046cb95492 +Author: Field G. Van Zee +Date: Wed Feb 11 19:33:10 2015 -0600 + + Fixed an obscure bug in 3mh/3m/4mh/4m packing. + + Details: + - Modified bli_packm_blk_var1.c and _var2.c to increase the triangular + case's panel increment by 1 if it would otherwise be odd. This is + particularly necessary in _var2.c when handling the interleaved 3m + or ro/io/rpi pack schemas, since division of an odd number by 2 can + happen if both the panel length and the panel packing dimension + (register packing blocksize) are odd, thus making their product odd. + - Modified bli_packm_init.c so that panel strides are increased by 1 + if they would otherwise be odd, even for non-3m related packing. + - Modified the trmm and trsm macro-kernels so that triangular packed + micro-panels are traversed with this new "increment by 1 if odd" + policy. + - Added sanity checks in trmm and trsm macro-kernels that would result + in an abort() if the conditions that would lead to a "divide odd + integer by 2" scenario ever manifest. + - Defined bli_is_odd(), _is_even() macros in bli_scalar_macro_defs.h. + +commit 650d2a6ff2e593151a296ca86b5214afcc747afc +Author: Field G. Van Zee +Date: Mon Feb 9 14:59:20 2015 -0600 + + Added initial support for imaginary stride. + + Details: + - Added an imaginary stride field ("is") to obj_t. + - Renamed bli_obj_set_incs() macro to bli_obj_set_strides(). + - Defined bli_obj_imag_stride() and bli_obj_set_imag_stride() and + added invocations in key locations. + - Added some basic error-checking related to imaginary stride. + - For now, imaginary stride will not be exposed into the most-used + BLIS APIs such as bli_obj_create(), and certainly not the + computational APIs such as bli_dgemm(). + +commit f05a57634a7c8e3864b25b3335d1194c1ea1aeb9 +Author: Field G. Van Zee +Date: Sun Feb 8 19:40:34 2015 -0600 + + Defined gemm cntl function to query ukrs func_t. + + Details: + - Added a new function, bli_gemm_cntl_ukrs(), that returns the func_t* + for the gemm micro-kernels from the leaf node of the control tree. + This allows all the func_t* fields from higher-level nodes in the tree + to be NULL, which makes the function that builds the control trees + slightly easier to read. + - Call bli_gemm_cntl_ukrs() instead of the cntl_gemm_ukrs() macro in + all bli_*_front() functions (which is needed to apply the row/column + preference optimization). + - In all level-3 bli_*_cntl_init() functions, changed the _obj_create() + function arguments corresponding to the gemm_ukrs fields in higher- + level cntl tree nodes to NULL. + - Removed some old her2k macro-kernels. + +commit cefd3d5d2001264de17cf63dae541f890cb9daaf +Author: Tyler Smith +Date: Thu Feb 5 11:09:12 2015 -0600 + + A couple of functions were incorrectly ifdeffed away on Xeon Phi. Fixed this + +commit 7574c9947d57a19f613880e3b9f62f8c8f6df4ec +Author: Field G. Van Zee +Date: Wed Feb 4 12:11:55 2015 -0600 + + Added basic flop-counting mechanism (level-3 only). + + Details: + - Added optional flop counting to all level-3 front-ends, which is + enabled via BLIS_ENABLE_FLOP_COUNT. The flop count can be + reset at any time via bli_flop_count_reset() and queried via + bli_flop_count(). Caveats: + - flop counts are approximate for her[2]k, syr[2]k, trmm, and + trsm operations; + - flop counts ignore extra flops due to non-unit alpha; + - flop counts do not account for situations where beta is zero. + +commit ceda4f27d1f1bcf19320e09848e0f2e3b9941e6c +Author: Field G. Van Zee +Date: Thu Jan 29 13:22:54 2015 -0600 + + Implemented bli_obj_imag_equals(). + + Details: + - Implemented a new function, bli_obj_imag_equals(), which compares the + imaginary part of the first argument to the second argument, which may + be a BLIS_CONSTANT or of a regular real datatype. + +commit 81114824a05a9053229efd577a8a94a856deda93 +Author: Field G. Van Zee +Date: Tue Jan 6 12:15:21 2015 -0600 + + Minor 4m/3m consolidation to mem_pool_macro_defs.h. + + Details: + - Merged the 4m and 3m definitions in bli_mem_pool_macro_defs.h to + reduce code and improve readability. + +commit 36a9b7b7436d9423ba4de2a9f85cfcd43577b783 +Author: Tyler Michael Smith +Date: Wed Dec 17 21:53:50 2014 +0000 + + reduced the default number of MC by KC blocks for bgq + +commit c60619c7c3568f044a849abbab60209aa7455423 +Author: Field G. Van Zee +Date: Tue Dec 16 17:08:22 2014 -0600 + + Minor tweaks for 3m4m test drivers. + + Details: + - Changed gemm_kc blocksizes to be reduced by two-thirds instead of + half. + - Changed 3m4m/test_gemm.c driver to divide by 3 instead of 2 when + computing the fixed k dimension. + - Fixed runme.sh so that it would use multiple threads for s/dgemm + cases. + +commit c6929ba6a5e6f633a7295e979a2b8df8c7ecdb1b +Author: Field G. Van Zee +Date: Tue Dec 16 11:27:50 2014 -0600 + + Added 4m_1b to test/3m4m test driver and script. + +commit 785d480805fc0d6f4251b5499933515740b6b2a7 +Merge: 9456f33 4156c08 +Author: Field G. Van Zee +Date: Fri Dec 12 14:34:19 2014 -0600 + + Merge branch 'master' of github.com:flame/blis + +commit 9456f330af4617f9ee32972d51f974aa2d84f97b +Author: Field G. Van Zee +Date: Fri Dec 12 14:31:57 2014 -0600 + + Added 4m_1b implementation for gemm. + + Details: + - Added yet another 4m-based implementation for complex domain level-3 + operations. This method, which the 3m/4m paper identifies as Algorithm + "4m_1b" fissures the first loop around the micro-kernel so that the + real sub-panel of the current micro-panel of B is multiplied against + (both sub-panels of) all micro-panels of A, before doing the same for + the imaginary sub-panel of the micro-panel of B. For now, only gemm is + supported, and 4m_1b (labeled "4mb" within the framework) is not yet + integrated into the test suite. + +commit 4156c0880d9aea4ff04a9c4fa139ba8c437d8bfb +Author: Field G. Van Zee +Date: Tue Dec 9 16:03:14 2014 -0600 + + Fixed obscure level-2 packing / general stride bug. + + Details: + - Fixed a bug in certain structured level-2 operations that manifested + only when the structured matrix was provided to BLIS as matrix stored + with general stride. The bug was introduced in c472993b when the + densify field was removed from the packm control tree node and + associated APIs. Since then, the packed object was unconditionally + marked with an uplo field of BLIS_DENSE. This is fine for level-3 + operations where micro-panels are always densified, but in level-2 + contexts, the underlying unblocked variant (fused or unfused) of + structured operations (e.g. trmv) still needs to know whether to + execute its "lower" or "upper" branches of code. Since this field + was unconditionally being set to BLIS_DENSE, the unblocked variants + were always executed the "else" branch, which happened to be the + "lower" case code. Thus, running an upper case produced the wrong + answer. This most obviously manifested in the form of failures for + trmm, trmm3, and trsm in the test suite. + The bug was fixed by setting the packed object's uplo field to + BLIS_DENSE only if the schema indicated that micro-panels were to be + packed. Otherwise, we can assume we are packing to regular row or + column storage, as is the case with level-2 packing. Thanks to + Francisco Igual for reporting the testsuite failures and ultimately + leading us to this bug. + +commit 689f60a578b461119e9ea90c74f642b9eb79addb +Merge: bef24e6 483e4d6 +Author: Field G. Van Zee +Date: Sun Dec 7 14:03:30 2014 -0600 + + Merge pull request #21 from figual/master + + Adding armv8a configuration and micro-kernels. + +commit 483e4d6a3fdbef9d9ab47fb674c9476c70ca9f0f +Author: Francisco D. Igual +Date: Sun Dec 7 20:27:49 2014 +0100 + + Adding armv8a configuration and micro-kernels. + + Only sgemm micro-kernel is fully functional at this point. + +commit bef24e67e0f93579c2a80315348dc2e227f72a72 +Author: Tyler Smith +Date: Wed Nov 26 18:00:56 2014 -0600 + + Fixed a type of race condition exposed by pthreads implementation. + Lead thread of the inner thread communicator could exit subproblem, move on the next iteration of the loop and modify a1_pack, b1_pack, or c1_pack while other threads were still using those. + + Barriers were inserted to fix this. + +commit 76bde44411f0e34266bab9d666a54ef22be97320 +Merge: e56e614 f3d729e +Author: Field G. Van Zee +Date: Wed Nov 26 17:25:24 2014 -0600 + + Merge branch 'master' of github.com:flame/blis + +commit f3d729e504ec012e7dc7e02b2ecd42e004c6894d +Author: Tyler Michael Smith +Date: Wed Nov 26 22:25:24 2014 -0600 + + Added static mutex to bli_init and bli_finalize + +commit d71cc797866ff502ad1127527016f463267eef80 +Author: Tyler Michael Smith +Date: Wed Nov 26 21:35:39 2014 -0600 + + Refactored bli_threading files and added support for pthreads + +commit e56e61438ff7fcf25a48c0b7603f18df782b50b6 +Author: Field G. Van Zee +Date: Wed Nov 26 17:20:35 2014 -0600 + + Minor cleanups to bli_threading.h and friends. + + Details: + - No longer need to define BLIS_ENABLE_MULTITHREADING manually in + bli_config.h; it now gets defined when BLIS_ENABLE_OPENMP or + BLIS_ENABLE_PTHREADS is defined. + - Added sanity check to prevent both BLIS__ENABLE_OPENMP and + BLIS_ENABLE_PTHREADS from being enabled simultaneously. + - Reorganization of bli_threading*.h header files, which led to + simplification of threading-related part of blis.h. + - added "-fopenmp -lpthread" to LDFLAGS of sandybridge make_defs.mk + file. + +commit 3be2744cbe2c56d38c23fd818aa5c1f10cc7ea51 +Author: Field G. Van Zee +Date: Fri Nov 21 12:28:08 2014 -0600 + + Update to template gemm ukernel comments. + + Details: + - Updated comments on alignment of a1 and b1 to match wiki. + +commit 994429c6881b2ade92d9d7949bcaebfbf2cc65eb +Merge: 58796ab 694029d +Author: Field G. Van Zee +Date: Thu Nov 20 13:55:35 2014 -0600 + + Merge pull request #20 from TimmyLiu/master + + #define PASTEF773 required by cblas compatibility layer + +commit 694029d9d7db857d642ab536955c0621791108c8 +Author: Timmy +Date: Wed Nov 19 15:25:14 2014 -0600 + + #define PASTEF773 required by cblas compatiility layer + +commit 58796abda66b133346f8d523b39178afc336351f +Author: Field G. Van Zee +Date: Thu Nov 6 14:31:52 2014 -0600 + + Removed KC constraint comments from _kernel.h files. + + Details: + - Since 4674ca8c, the constraint that KC be a multiple of both MR and + NR have been relaxed, and thus it was time to remove the comments + from the top of the bli_kernel.h files of all configurations. + +commit 7bbc95a54f706d43c7f7951f0e5995f86130cd52 +Author: Field G. Van Zee +Date: Wed Oct 29 10:52:23 2014 -0500 + + Added new piledriver micro-kernels. + + Details: + - Added new micro-kernels for the AMD piledriver architecture (one + for each datatype). + - Updates and tweaks to piledriver configuration. + - Added 3xk packm micro-kernel support. + - Explicitly unrolled some of the smaller packm micro-kernels. + - Added notes to avx/sandybridge and piledriver micro-kernel files + acknowledging the influence of the corresponding kernel code in + OpenBLAS. + +commit 59613f1d5500f6279963327db2fbc84bc9135183 +Author: Field G. Van Zee +Date: Thu Oct 23 17:21:37 2014 -0500 + + Added separeate micro-panel alignment for A and B. + + Details: + - Changed the recently-added micro-panel alignment macros so that we now + have two sets--one for micro-panels of matrix A and one for micro- + panels of matrix B: BLIS_UPANEL_[AB]_ALIGN_SIZE_?. + - Store each set of alignment values into a separate blksz_t object in + bli_gemm_cntl_init(). + - Adjusted packm_init() to use the separate alignment values. + - Added query routines for the new alignment values to bli_info.c. + - Modified test suite output accordingly. + +commit a8e12884ee1fddd3fd77ca5a68aa0cb857f3af57 +Author: Field G. Van Zee +Date: Thu Oct 23 11:35:48 2014 -0500 + + CHANGELOG update (0.1.6) + +commit 38ea5022e4ed846112198c4e1672fcdaeb90dc71 (tag: 0.1.6) Author: Field G. Van Zee Date: Thu Oct 23 11:35:45 2014 -0500 Version file update (0.1.6) -commit a3e6341bdb0e28411f935d6b4708a6389663e004 (origin/master, origin/HEAD) +commit a3e6341bdb0e28411f935d6b4708a6389663e004 Author: Field G. Van Zee Date: Thu Oct 23 11:13:28 2014 -0500 @@ -948,7 +1811,7 @@ Date: Mon Aug 4 16:01:59 2014 -0500 CHANGELOG update (0.1.5) -commit bde56d0ecfd0ec20330fac290b91a6dca0cf94e9 (0.1.5) +commit bde56d0ecfd0ec20330fac290b91a6dca0cf94e9 (tag: 0.1.5) Author: Field G. Van Zee Date: Mon Aug 4 16:01:58 2014 -0500 @@ -1097,7 +1960,7 @@ Date: Sun Jul 27 18:20:13 2014 -0500 CHANGELOG update (0.1.4) -commit a7537071b152ecff671f8716595d37dc09e4fd51 (0.1.4) +commit a7537071b152ecff671f8716595d37dc09e4fd51 (tag: 0.1.4) Author: Field G. Van Zee Date: Sun Jul 27 18:20:12 2014 -0500 @@ -1496,7 +2359,7 @@ Date: Mon Jun 23 13:48:17 2014 -0500 CHANGELOG update (0.1.3) -commit 036cc634918463b1caa0fd89c9a211f2f5639af7 (0.1.3) +commit 036cc634918463b1caa0fd89c9a211f2f5639af7 (tag: 0.1.3) Author: Field G. Van Zee Date: Mon Jun 23 13:48:17 2014 -0500 @@ -1629,7 +2492,7 @@ Date: Thu Jun 5 10:54:16 2014 -0500 CHANGELOG update (for 0.1.2). -commit 00f232f8ed1f7c41619b12ebf779ebe2c3b2d3cd (0.1.2) +commit 00f232f8ed1f7c41619b12ebf779ebe2c3b2d3cd (tag: 0.1.2) Author: Tyler Smith Date: Mon Jun 2 13:40:57 2014 -0500 @@ -2257,7 +3120,7 @@ Date: Tue Feb 25 17:58:42 2014 -0600 CHANGELOG update (for 0.1.1). -commit fde5f1fdece19881f50b142e8611b772a647e6d2 (0.1.1) +commit fde5f1fdece19881f50b142e8611b772a647e6d2 (tag: 0.1.1) Author: Field G. Van Zee Date: Tue Feb 25 13:34:56 2014 -0600 @@ -3171,7 +4034,7 @@ Date: Mon Nov 11 10:15:40 2013 -0600 CHANGELOG update (for 0.1.0). -commit 089048d5895a30221b6b1976c9be93ad6443420d (0.1.0) +commit 089048d5895a30221b6b1976c9be93ad6443420d (tag: 0.1.0) Author: Field G. Van Zee Date: Sat Nov 9 17:18:00 2013 -0600 @@ -3886,7 +4749,7 @@ Date: Fri Jul 19 17:15:03 2013 -0500 CHANGELOG update (for 0.0.9). -commit 0680916fdd532f7a4716b11a2515243b2c08d00f (0.0.9) +commit 0680916fdd532f7a4716b11a2515243b2c08d00f (tag: 0.0.9) Author: Field G. Van Zee Date: Thu Jul 18 18:04:34 2013 -0500 @@ -4124,7 +4987,7 @@ Date: Wed Jun 12 16:40:04 2013 -0500 CHANGELOG update. -commit 5b641c3bab31eac6a1795b9f6e3f86c59651ca50 (0.0.8) +commit 5b641c3bab31eac6a1795b9f6e3f86c59651ca50 (tag: 0.0.8) Author: Field G. Van Zee Date: Wed Jun 12 16:02:12 2013 -0500 @@ -4311,7 +5174,7 @@ Date: Wed May 1 15:00:30 2013 -0500 CHANGELOG update. -commit 6bfa96f84887dec0b4cf8be5d38dd634c2f8951d (0.0.7) +commit 6bfa96f84887dec0b4cf8be5d38dd634c2f8951d (tag: 0.0.7) Author: Field G. Van Zee Date: Tue Apr 30 19:35:54 2013 -0500 @@ -4693,7 +5556,7 @@ Date: Sat Apr 13 16:53:16 2013 -0500 CHANGELOG update. -commit ec16c52f2ecf419c749175ce0a297441c10f1c68 (0.0.6) +commit ec16c52f2ecf419c749175ce0a297441c10f1c68 (tag: 0.0.6) Author: Field G. Van Zee Date: Sat Apr 13 16:41:16 2013 -0500 @@ -5003,7 +5866,7 @@ Date: Sun Mar 24 20:18:12 2013 -0500 CHANGELOG update. -commit b65cdc57d9e51fa00e3c03539cfb7e045707d0f4 (0.0.5) +commit b65cdc57d9e51fa00e3c03539cfb7e045707d0f4 (tag: 0.0.5) Author: Field G. Van Zee Date: Sun Mar 24 20:01:49 2013 -0500 @@ -5107,7 +5970,7 @@ Date: Mon Mar 18 10:37:03 2013 -0500 CHANGELOG update. -commit e7d41229d3b1674e74f47d7f29fae004a745201a (0.0.4) +commit e7d41229d3b1674e74f47d7f29fae004a745201a (tag: 0.0.4) Author: Field G. Van Zee Date: Fri Mar 15 17:12:36 2013 -0500 @@ -5235,7 +6098,7 @@ Date: Fri Feb 22 12:38:45 2013 -0600 configuration directory (bl2_config.h, specifically) given that it can be expected to be tweaked by some developers. -commit ede75693e5a36c6006087c4a7df834175b604504 (0.0.3) +commit ede75693e5a36c6006087c4a7df834175b604504 (tag: 0.0.3) Author: Field G. Van Zee Date: Fri Feb 22 12:11:24 2013 -0600 @@ -5445,7 +6308,7 @@ Date: Mon Feb 11 13:38:07 2013 -0600 CHANGELOG update. -commit 768fcebaa8be0eb936a6e7a02cd8a19438c79d99 (0.0.2) +commit 768fcebaa8be0eb936a6e7a02cd8a19438c79d99 (tag: 0.0.2) Author: Field G. Van Zee Date: Mon Feb 11 13:20:44 2013 -0600 @@ -5687,7 +6550,7 @@ Date: Mon Dec 10 17:23:32 2012 -0600 Minor updates towards to 0.0.1. -commit 7ad4ebef38b8e6eea9b6091844ba7294ec870271 (0.0.1) +commit 7ad4ebef38b8e6eea9b6091844ba7294ec870271 (tag: 0.0.1) Author: Field G. Van Zee Date: Mon Dec 10 16:18:40 2012 -0600 @@ -5755,7 +6618,7 @@ Date: Thu Dec 6 14:27:11 2012 -0600 Wrote first draft of INSTALL file. -commit bcbe81235a35ccfdbcc2f2319a0ca6e04f75a785 (0.0.0) +commit bcbe81235a35ccfdbcc2f2319a0ca6e04f75a785 (tag: 0.0.0) Author: Field G. Van Zee Date: Thu Dec 6 12:42:35 2012 -0600 diff --git a/CREDITS b/CREDITS index 79c97634e4..eea4cdee7e 100644 --- a/CREDITS +++ b/CREDITS @@ -12,9 +12,9 @@ but many others have contributed input and feedback, including: Vernon Austel (IBM, T.J. Watson Research Center) Jed Brown (Argonne National Laboratory) John Gunnels (IBM, T.J. Watson Research Center) - Jeff Hammond (Argonne National Laboratory) + Jeff Hammond (Intel) Francisco Igual (Universidad Complutense de Madrid) - Lee Killough (Appro International) + Lee Killough (Cray) Mike Kistler (IBM, Austin Research Laboratory) Bryan Marker (The University of Texas at Austin) Devin Matthews (The University of Texas at Austin) @@ -33,6 +33,7 @@ BLIS to new architectures as proofs-of-concept: Tyler Smith (The University of Texas at Austin) Zhang Xianyi (Chinese Academy of Sciences) -BLIS's development was partially funded by grants from Microsoft and the -National Science Foundation (NSF Awards CCF-0917167 and OCI-1148125). +BLIS's development was partially funded by grants from Microsoft, Intel, +Texas Instruments, AMD, and the National Science Foundation (NSF Awards +CCF-0917167, ACI-1148125/1340293, and CCF-1320112). diff --git a/Makefile b/Makefile index 9ad0d75686..c9fe957b51 100644 --- a/Makefile +++ b/Makefile @@ -633,6 +633,11 @@ ifeq ($(BLIS_ENABLE_VERBOSE_MAKE_OUTPUT),yes) ./$(TESTSUITE_BIN) -g $(TESTSUITE_CONF_GEN_PATH) \ -o $(TESTSUITE_CONF_OPS_PATH) \ > $(TESTSUITE_OUT_FILE) + +else ifeq ($(BLIS_ENABLE_TEST_OUTPUT), yes) + ./$(TESTSUITE_BIN) -g $(TESTSUITE_CONF_GEN_PATH) \ + -o $(TESTSUITE_CONF_OPS_PATH) | \ + tee $(TESTSUITE_OUT_FILE) else @echo "Running $(TESTSUITE_BIN) with output redirected to '$(TESTSUITE_OUT_FILE)'" @./$(TESTSUITE_BIN) -g $(TESTSUITE_CONF_GEN_PATH) \ diff --git a/README b/README deleted file mode 100644 index 7ce5513dc5..0000000000 --- a/README +++ /dev/null @@ -1,49 +0,0 @@ - -BLIS framework -README ---- - -Thank you for deciding to try out the BLIS framework! - -BLIS is a portable framework for instantiating BLAS-like libraries. The -framework was designed to isolate essential kernels of computation that, -when optimized, immediately enable optimized implementations of most of -its commonly used and computationally intensive operations. - -BLIS has many features. For more detailed information about the project, -please check the BLIS homepage: - - http://code.google.com/p/blis/ - -You can keep in touch with developers and other users of the project by -joining one or more of the following mailing lists: - - o blis-announce - http://groups.google.com/group/blis-announce - Used only for announcements and other important messages regarding - BLIS. - - o blis-discuss - http://groups.google.com/group/blis-discuss - Please join and post to this mailing list if you have general questions - or feedback regarding BLIS. Application developers (end users) should - probably post here. - - o blis-devel - http://groups.google.com/group/blis-devel - Please join and post to this mailing list if you are a BLIS developer - (i.e., you are trying to use BLIS to create libraries, you want to - write kernels for the framework, or you are trying to modify or extend - the framework itself). - -Also, please read the LICENSE file for information on copying and -distributing this software. - -For a step-by-step guide on configuring, compiling, and installing BLIS, -please read the INSTALL file. Also, please check the BLIS website's wiki -page for other useful how-to guides. - -Thanks again for your interest in BLIS! - -Regards, - -Field G. Van Zee -field@cs.utexas.edu - diff --git a/README.md b/README.md new file mode 100644 index 0000000000..83920d5fd0 --- /dev/null +++ b/README.md @@ -0,0 +1,277 @@ +![The BLIS cat is sleeping.](http://www.cs.utexas.edu/users/field/blis_cat.png) + +[![Build Status](https://travis-ci.org/flame/blis.svg?branch=master)](https://travis-ci.org/flame/blis) + +Introduction +------------ + +BLIS is a portable software framework for instantiating high-performance +BLAS-like dense linear algebra libraries. The framework was designed to +isolate essential kernels of computation that, when optimized, immediately +enable optimized implementations of most of its commonly used and +computationally intensive operations. BLIS is written in [ISO +C99](http://en.wikipedia.org/wiki/C99) and available under a +[new/modified/3-clause BSD +license](http://opensource.org/licenses/BSD-3-Clause). While BLIS exports a +[new BLAS-like API](), it also includes a BLAS compatibility layer which gives +application developers access to BLIS implementations via traditional [BLAS +routine calls](http://www.netlib.org/lapack/lug/node145.html). + +For a thorough presentation of our framework, please read our recently accepted +journal article, ["BLIS: A Framework for Rapidly Instantiating BLAS +Functionality"](http://www.cs.utexas.edu/users/flame/pubs/blis1_toms_rev3.pdf). +For those who just want an executive summary, please see the next section. + +In a follow-up article, ["The BLIS Framework: Experiments in +Portability"](http://www.cs.utexas.edu/users/flame/pubs/blis2_toms_rev3.pdf), +we investigate using BLIS to instantiate level-3 BLAS implementations on a +variety of general-purpose, low-power, and multicore architectures. + +An IPDPS'14 conference paper titled ["Anatomy of High-Performance Many-Threaded +Matrix +Multiplication"](http://www.cs.utexas.edu/users/flame/pubs/blis3_ipdps14.pdf) +systematically explores the opportunities for parallelism within the five loops +that BLIS exposes in its matrix multiplication algorithm. + +It is our belief that BLIS offers substantial benefits in productivity when +compared to conventional approaches to developing BLAS libraries, as well as a +much-needed refinement of the BLAS interface, and thus constitutes a major +advance in dense linear algebra computation. While BLIS remains a +work-in-progress, we are excited to continue its development and further +cultivate its use within the community. + +Key Features +------------ + +BLIS offers several advantages over traditional BLAS libraries: + + * **Portability that doesn't impede high performance.** Portability was a top +priority of ours when creating BLIS. With zero additional effort on the part of +the developer, BLIS is configurable as a fully-functional reference +implementation. But more importantly, the framework identifies and isolates a +key set of computational kernels which, when optimized, immediately and +automatically optimize performance across virtually all level-2 and level-3 +BLIS operations. In this way, the framework acts as a productivity multiplier. +And since the optimized (non-portable) code is compartmentalized within these +few kernels, instantiating a high-performance BLIS library on a new +architecture is a relatively straightforward endeavor. + + * **Generalized matrix storage.** The BLIS framework exports interfaces that +allow one to specify both the row stride and column stride of a matrix. This +allows one to compute with matrices stored in column-major order, row-major +order, or by general stride. (This latter storage format is important for those +seeking to implement tensor contractions on multidimensional arrays.) +Furthermore, since BLIS tracks stride information for each matrix, operands of +different storage formats can be used within the same operation invocation. By +contrast, BLAS requires column-major storage. And while the CBLAS interface +supports row-major storage, it does not allow mixing storage formats. + + * **Full support for the complex domain.** BLIS operations are developed and +expressed in their most general form, which is typically in the complex domain. +These formulations then simplify elegantly down to the real domain, with +conjugations becoming no-ops. Unlike the BLAS, all input operands in BLIS that +allow transposition and conjugate-transposition also support conjugation +(without transposition), which obviates the need for thread-unsafe workarounds. +Also, where applicable, both complex symmetric and complex Hermitian forms are +supported. (BLAS omits some complex symmetric operations, such as `symv`, +`syr`, and `syr2`.) + + * **Advanced multithreading support.** BLIS allows multiple levels of +symmetric multithreading for nearly all level-3 operations. (Currently, users +may choose to obtain parallelism via either OpenMP or POSIX threads). This +means that matrices may be partitioned in multiple dimensions simultaneously to +attain scalable, high-performance parallelism on multicore and many-core +architectures. The key to this innovation is a thread-specific control tree +infrastructure which encodes information about the logical thread topology and +allows threads to query and communicate data amongst one another. BLIS also +employs so-called "quadratic partitioning" when computing dimension sub-ranges +for each thread, so that arbitrary diagonal offsets of structured matrices with +unreferenced regions are taken into account to achieve proper load balance. + + * **Ease of use.** The BLIS framework, and the library of routines it +generates, are easy to use for end users, experts, and vendors alike. An +optional BLAS compatibility layer provides application developers with +backwards compatibility to existing BLAS-dependent codes. Or, one may adjust or +write their application to take advantage of new BLIS functionality (such as +generalized storage formats or additional complex operations) by calling BLIS +directly. BLIS's interfaces will feel familiar to many veterans of BLAS since +BLIS exports APIs with BLAS-like calling sequences. And experts will find +BLIS's internal object-based APIs a delight to use when customizing or writing +their own BLIS operations. (Objects are relatively lightweight `structs` and +passed by address, which helps tame function calling overhead.) + + * **Multilayered API and exposed kernels.** The BLIS framework exposes its +implementations in various layers, allowing expert developers to access exactly +the functionality desired. This layered interface includes that of the +lowest-level kernels, for those who wish to bypass the bulk of the framework. +Optimizations can occur at various levels, in part thanks to exposed packing +and unpacking facilities, which by default are highly parameterized and +flexible. + + * **Functionality that grows with the community's needs.** As its name +suggests, the BLIS framework is not a single library or static API, but rather +a nearly-complete template for instantiating high-performance BLAS-like +libraries. Furthermore, the framework is extensible, allowing developers to +leverage existing components to support new operations as they are identified. +If such operations require new kernels for optimal efficiency, the framework +and its APIs will be adjusted and extended accordingly. + + * **Code re-use.** Auto-generation approaches to achieving the aforementioned +goals tend to quickly lead to code bloat due to the multiple dimensions of +variation supported: operation (i.e. `gemm`, `herk`, `trmm`, etc.); parameter +case (i.e. side, [conjugate-]transposition, upper/lower storage, unit/non-unit +diagonal); datatype (i.e. single-/double-precision real/complex); matrix +storage (i.e. row-major, column-major, generalized); and algorithm (i.e. +partitioning path and kernel shape). These "brute force" approaches often +consider and optimize each operation or case combination in isolation, which is +less than ideal when the goal is to provide entire libraries. BLIS was designed +to be a complete framework for implementing basic linear algebra operations, +but supporting this vast amount of functionality in a manageable way required a +holistic design that employed careful abstractions, layering, and recycling of +generic (highly parameterized) codes, subject to the constraint that high +performance remain attainable. + + * **A foundation for mixed domain and/or mixed precision operations.** BLIS +was designed with the hope of one day allowing computation on real and complex +operands within the same operation. Similarly, we wanted to allow mixing +operands' floating-point precisions, or both domain and precision. +Unfortunately, this feature results in a significant amount of additional code, +mostly in level-2 and lower operations, thus, it is disabled by default. +However, mixing domains in level-3 operations is possible, in theory, with +almost no additional effort on the part of the library developer, and such +operations would remain capable of high performance. (Please note that this +functionality is still highly experimental and should be thought of as a +feature that will be more thoroughly implemented at some future date.) + +Getting Started +--------------- + +If you just want to browse a quick-reference guide on user-level BLIS +interfaces, please read the [BLIS API quick +reference](https://github.com/flame/blis/wiki/BLISAPIQuickReference). +There you will find a brief description of each operation as well as some more +general information needed when developing an application with BLIS. + +Have a quick question? You may find the answer in our list of [frequently asked +questions](https://github.com/flame/blis/wiki/FAQ). + +Does BLIS contain kernels optimized for your favorite architecture? Please see +our [Hardware Support wiki](https://github.com/flame/blis/wiki/HardwareSupport) +for a full list of optimized kernels. + +We also provide wikis on the following topics, which will likely be of interest +to many users and developers: + * [Build system](https://github.com/flame/blis/wiki/BuildSystem). +This wiki provides step-by-step instructions for building a BLIS library. +(Reminder: While BLIS supports configure-time hardware detection for certain +architectures, you may need to manually specify a configuration to use.) + * [Configuration](https://github.com/flame/blis/wiki/ConfigurationHowTo). +This wiki describes how to create a BLIS "configuration", which captures all of +the details necessary to build BLIS for a specific hardware architecture. +Configurations specify things like blocksizes, kernel names, and various +optional configuration settings. + * [Kernels](https://github.com/flame/blis/wiki/KernelsHowTo). +This wiki describes each of the BLIS kernel operations in detail and should +provide developers with most of the information needed to get started with +writing and optimizing their own kernels. + * [Test suite](https://github.com/flame/blis/wiki/Testsuite). +This wiki contains detailed instructions on running the BLIS test suite, +located in the top-level directory testsuite. + +Discussion +---------- + +You can keep in touch with developers and other users of the project by joining +one of the following mailing lists: + + * [blis-discuss](http://groups.google.com/group/blis-discuss): Please join and +post to this mailing list if you have general questions or feedback regarding +BLIS. Application developers (end users) should probably post here, unless they +have bug reports, in which case they should post to +[blis-devel](http://groups.google.com/group/blis-devel). + + * [blis-devel](http://groups.google.com/group/blis-devel): Please join and +post to this mailing list if you are a BLIS developer (i.e., you are trying to +use BLIS to create libraries, you want to write kernels for the framework, or +you are trying to modify or extend the framework itself). Also, if you would +like to submit a bug report, or discuss a possible bug, please use this list. +**Note:** Most of the interesting discussions happen here; don't be afraid to +join! + +Citations +--------- + +For those of you looking for the appropriate article to cite regarding BLIS, we +recommend citing our +[first ACM TOMS journal paper](http://www.cs.utexas.edu/users/flame/pubs/blis1_toms_rev3.pdf): + +``` +@article{BLIS1, + author = {Field G. {V}an~{Z}ee and Robert A. {v}an~{d}e~{G}eijn}, + title = {{BLIS}: A Framework for Rapidly Instantiating {BLAS} Functionality}, + journal = {ACM Transactions on Mathematical Software}, + volume = {41}, + number = {3}, + pages = {14:1--14:33}, + year = {2015}, + issue_date = {June 2015}, + url = {http://doi.acm.org/10.1145/2764454}, +} +``` + +You may also cite the +[second ACM TOMS journal paper](http://www.cs.utexas.edu/users/flame/pubs/blis2_toms_rev3.pdf): + +``` +@article{BLIS2, + author = {Field G. {V}an~{Z}ee and Tyler Smith and Francisco D. Igual and + Mikhail Smelyanskiy and Xianyi Zhang and Michael Kistler and Vernon Austel and + John Gunnels and Tze Meng Low and Bryan Marker and Lee Killough and + Robert A. {v}an~{d}e~{G}eijn}, + title = {The {BLIS} Framework: Experiments in Portability}, + journal = {ACM Transactions on Mathematical Software}, + year = 2015, + note = {Accepted}, +} +``` + +We also have a third paper, submitted to IPDPS 2014, on achieving +[multithreaded parallelism in BLIS](http://www.cs.utexas.edu/users/flame/pubs/blis3_ipdps14.pdf): + +``` +@inproceedings{BLIS3, + author = {Tyler M. Smith and Robert A. {v}an~{d}e~{G}eijn and Mikhail Smelyanskiy and + Jeff R. Hammond and Field G. {V}an~{Z}ee}, + title = {Anatomy of High-Performance Many-Threaded Matrix Multiplication}, + booktitle = {28th IEEE International Parallel \& Distributed Processing Symposium + (IPDPS 2014)}, + year = 2014, +} +``` + +A fourth paper, submitted to ACM TOMS, also exists, which proposes an +[analytical model](http://www.cs.utexas.edu/users/flame/pubs/TOMS-BLIS-Analytical.pdf) for determining blocksize parameters in BLIS: + +``` +@article{BLIS4, + author = {Tze Meng Low and Francisco D. Igual and Tyler M. Smith and Enrique S. Quintana-Ort\'{\i}}, + title = {Analytical Models for the {BLIS} Framework}, + journal = {ACM Transactions on Mathematical Software}, + year = 2015, + note = {Pending}, +} +``` + +Funding +------- + +This project and its associated research was partially sponsored by grants from +[Microsoft](http://www.microsoft.com/), [Intel](http://www.intel.com/), [Texas +Instruments](http://www.ti.com/), and [AMD](http://www.amd.com/), as well as +grants from the [National Science Foundation](http://www.nsf.gov/) (Awards +CCF-0917167 ACI-1148125/1340293, and CCF-1320112). + +_Any opinions, findings and conclusions or recommendations expressed in this +material are those of the author(s) and do not necessarily reflect the views of +the National Science Foundation (NSF)._ + diff --git a/build/auto-detect/cpuid_x86.c b/build/auto-detect/cpuid_x86.c index e7757862b9..9a703c0855 100644 --- a/build/auto-detect/cpuid_x86.c +++ b/build/auto-detect/cpuid_x86.c @@ -42,13 +42,15 @@ #define CPUNAME_REFERENCE 0 #define CPUNAME_DUNNINGTON 1 #define CPUNAME_SANDYBRIDGE 2 -#define CPUNAME_BULLDOZER 3 -#define CPUNAME_PILEDRIVER 4 +#define CPUNAME_HASWELL 3 +#define CPUNAME_BULLDOZER 4 +#define CPUNAME_PILEDRIVER 5 static char *cpuname[] = { "reference", "dunnington", "sandybridge", + "haswell", "bulldozer", "piledriver", }; @@ -122,73 +124,89 @@ int cpu_detect() int eax, ebx, ecx, edx; int vendor, family, extend_family, model, extend_model; - if (!have_cpuid()) return CPUNAME_REFERENCE; + if ( !have_cpuid() ) return CPUNAME_REFERENCE; - vendor=get_vendor(); + vendor = get_vendor(); - cpuid(1, &eax, &ebx, &ecx, &edx); - extend_family = BITMASK(eax, 20, 0xff); - extend_model=BITMASK(eax, 16, 0x0f); - family=BITMASK(eax, 8, 0x0f); - model=BITMASK(eax, 4, 0x0f); + cpuid( 1, &eax, &ebx, &ecx, &edx ); + + extend_family = BITMASK( eax, 20, 0xff ); + extend_model = BITMASK( eax, 16, 0x0f ); + family = BITMASK( eax, 8, 0x0f ); + model = BITMASK( eax, 4, 0x0f ); if (vendor == VENDOR_INTEL){ switch (family) { case 0x6: switch (extend_model) { case 1: - switch (model) { - case 7: - //penryn uses dunnington config. - return CPUNAME_DUNNINGTON; - case 13: - return CPUNAME_DUNNINGTON; - } - break; + switch (model) { + case 7: + //penryn uses dunnington config. + return CPUNAME_DUNNINGTON; + case 13: + return CPUNAME_DUNNINGTON; + } + break; case 2: - switch (model) { - case 10: - case 13: - if(support_avx()) { - return CPUNAME_SANDYBRIDGE; - }else{ - return CPUNAME_REFERENCE; //OS doesn't support AVX - } - } - break; + switch (model) { + case 10: + case 13: + if(support_avx()) { + return CPUNAME_SANDYBRIDGE; + }else{ + return CPUNAME_REFERENCE; //OS doesn't support AVX + } + } + break; case 3: - switch (model) { - case 10: - case 14: - //Ivy Bridge - if(support_avx()) { - return CPUNAME_SANDYBRIDGE; - }else{ - return CPUNAME_REFERENCE; //OS doesn't support AVX - } - case 12: - case 15: - //Haswell. Temp use Sandy Brdige - if(support_avx()) { - return CPUNAME_SANDYBRIDGE; - }else{ - return CPUNAME_REFERENCE; //OS doesn't support AVX - } - - } - break; + switch (model) { + case 10: + case 14: + //Ivy Bridge + if(support_avx()) { + return CPUNAME_SANDYBRIDGE; + }else{ + return CPUNAME_REFERENCE; //OS doesn't support AVX + } + case 12: + case 15: + //Haswell + case 13: //Broadwell + if(support_avx()) { + return CPUNAME_HASWELL; + }else{ + return CPUNAME_REFERENCE; //OS doesn't support AVX + } + + } + break; case 4: switch (model) { case 5: + case 6: + //Haswell + case 7: + case 15: + //Broadwell + if(support_avx()) { + return CPUNAME_HASWELL; + }else{ + return CPUNAME_REFERENCE; //OS doesn't support AVX + } + } + break; + case 5: + switch (model) { case 6: - //Haswell. Temp use Sandy Brdige - if(support_avx()) { - return CPUNAME_SANDYBRIDGE; - }else{ - return CPUNAME_REFERENCE; //OS doesn't support AVX - } + //Broadwell + if(support_avx()) { + return CPUNAME_HASWELL; + }else{ + return CPUNAME_REFERENCE; //OS doesn't support AVX + } } - break; + break; } break; } @@ -197,24 +215,24 @@ int cpu_detect() case 0xf: switch (extend_family) { case 6: - switch (model) { - case 1: - if(support_avx()) - return CPUNAME_BULLDOZER; - else - return CPUNAME_REFERENCE; //OS don't support AVX. - case 2: - if(support_avx()) - return CPUNAME_PILEDRIVER; - else - return CPUNAME_REFERENCE; //OS don't support AVX. - case 0: - //Steamroller. Temp use Piledriver. - if(support_avx()) - return CPUNAME_PILEDRIVER; - else - return CPUNAME_REFERENCE; //OS don't support AVX. - } + switch (model) { + case 1: + if(support_avx()) + return CPUNAME_BULLDOZER; + else + return CPUNAME_REFERENCE; //OS don't support AVX. + case 2: + if(support_avx()) + return CPUNAME_PILEDRIVER; + else + return CPUNAME_REFERENCE; //OS don't support AVX. + case 0: + //Steamroller. Temp use Piledriver. + if(support_avx()) + return CPUNAME_PILEDRIVER; + else + return CPUNAME_REFERENCE; //OS don't support AVX. + } } break; } diff --git a/build/check-test.sh b/build/check-test.sh new file mode 100755 index 0000000000..6277ada452 --- /dev/null +++ b/build/check-test.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# +# BLIS +# An object-based framework for developing high-performance BLAS-like +# libraries. +# +# Copyright (C) 2015, The University of Texas at Austin +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# - Neither the name of The University of Texas at Austin nor the names +# of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# +# check-test.sh +# +# Zhang Xianyi +# + + +grep -q FAILURE $1 + +if [ $? -eq 0 ]; then + echo "Test Failure" + exit 1 +else + echo "Test Pass" + exit 0 +fi \ No newline at end of file diff --git a/config/armv7a/bli_config.h b/config/armv7a/bli_config.h index a757069e1f..21041fc2a4 100644 --- a/config/armv7a/bli_config.h +++ b/config/armv7a/bli_config.h @@ -36,142 +36,8 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. #define BLIS_INT_TYPE_SIZE 32 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 1 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 32 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. -#define BLIS_SIMD_ALIGN_SIZE 32 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. #define BLIS_BLAS2BLIS_INT_TYPE_SIZE 32 -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - - #endif diff --git a/config/armv8a/bli_config.h b/config/armv8a/bli_config.h index 72cc930e4d..127a2bca70 100644 --- a/config/armv8a/bli_config.h +++ b/config/armv8a/bli_config.h @@ -36,142 +36,9 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. #define BLIS_INT_TYPE_SIZE 64 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 1 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. #define BLIS_SIMD_ALIGN_SIZE 16 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. #define BLIS_BLAS2BLIS_INT_TYPE_SIZE 32 -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - - #endif diff --git a/config/armv8a/bli_kernel.h b/config/armv8a/bli_kernel.h index 2e27051226..3bd7da7222 100644 --- a/config/armv8a/bli_kernel.h +++ b/config/armv8a/bli_kernel.h @@ -51,12 +51,12 @@ // (b) MR (for zero-padding purposes when MR and NR are "swapped") // -#define BLIS_DEFAULT_MC_S 416 // 1280 //160 // 160 // 160 //2048 //336 -#define BLIS_DEFAULT_KC_S 704 //1280 //672 //528 // 856 //2048 //528 +#define BLIS_DEFAULT_MC_S 336 +#define BLIS_DEFAULT_KC_S 336 #define BLIS_DEFAULT_NC_S 4096 -#define BLIS_DEFAULT_MC_D 80 //176 -#define BLIS_DEFAULT_KC_D 336 //368 +#define BLIS_DEFAULT_MC_D 160 +#define BLIS_DEFAULT_KC_D 304 #define BLIS_DEFAULT_NC_D 4096 #define BLIS_DEFAULT_MC_C 64 @@ -132,8 +132,6 @@ //#define BLIS_PACKDIM_MR_Z (BLIS_DEFAULT_MR_Z + ...) //#define BLIS_PACKDIM_NR_Z (BLIS_DEFAULT_NR_Z + ...) - - // -- LEVEL-2 KERNEL CONSTANTS ------------------------------------------------- diff --git a/config/armv8a/make_defs.mk b/config/armv8a/make_defs.mk index e81dc50260..63c03c6a09 100644 --- a/config/armv8a/make_defs.mk +++ b/config/armv8a/make_defs.mk @@ -76,20 +76,18 @@ GIT_LOG := $(GIT) log --decorate # # --- Determine the C compiler and related flags --- -##CC := gcc -CC := aarch64-linux-gnu-gcc -#CC := arm-linux-gnueabihf-gcc-4.9.2 +CC := gcc + # Enable IEEE Standard 1003.1-2004 (POSIX.1d). # NOTE: This is needed to enable posix_memalign(). -CPPROCFLAGS := -D_POSIX_C_SOURCE=200112L -#CMISCFLAGS := -std=c99 -mtune=cortex-a57 -mfpu=neon-fp-armv8 -march=armv8-a #-mfloat-abi=hard -mfpu=neon -CMISCFLAGS := -std=c99 -march=armv8-a+fp+simd -ftree-vectorize -O3 -mcpu=cortex-a57 -mtune=cortex-a57 #-mtune=cortex-a57 -march=armv8-a -mfloat-abi=hard -mfpu=neon +CPPROCFLAGS := -D_GNU_SOURCE +CMISCFLAGS := -std=c99 -march=armv8-a+fp+simd -ftree-vectorize -O3 -fopenmp -mcpu=cortex-a57.cortex-a53 -mtune=cortex-a57.cortex-a53 CPICFLAGS := -fPIC -CDBGFLAGS := -g +CDBGFLAGS := -g #-g3 -gdwarf-2 CWARNFLAGS := -Wall -COPTFLAGS := -march=armv8-a+fp+simd -ftree-vectorize -O3 -mcpu=cortex-a57 -mtune=cortex-a57 #-march=armv8-a -O2 -mtune=cortex-a57 -mfpu=neon-fp-armv8 #-mfpu=neon -O2 +COPTFLAGS := -march=armv8-a+fp+simd -ftree-vectorize -O3 -mcpu=cortex-a57.cortex-a53 -mtune=cortex-a57.cortex-a53 CKOPTFLAGS := $(COPTFLAGS) -CVECFLAGS := -march=armv8-a+fp+simd -ftree-vectorize -O3 -mcpu=cortex-a57 -mtune=cortex-a57 #-march=armv8-a -O2 -mtune=cortex-a57 -mfpu=neon-fp-armv8 +CVECFLAGS := #-march=armv8-a+fp+simd -ftree-vectorize -O3 -mcpu=cortex-a57.cortex-a53 -mtune=cortex-a57.cortex-a53 #-march=armv8-a -O2 -mtune=cortex-a57 -mfpu=neon-fp-armv8 # Aggregate all of the flags into multiple groups: one for standard # compilation, and one for each of the supported "special" compilation @@ -105,7 +103,7 @@ ARFLAGS := cru # --- Determine the linker and related flags --- LINKER := $(CC) SOFLAGS := -shared -LDFLAGS := -lm +LDFLAGS := -lm -fopenmp diff --git a/config/bgq/bli_config.h b/config/bgq/bli_config.h index 09396b6a01..9810b5c116 100644 --- a/config/bgq/bli_config.h +++ b/config/bgq/bli_config.h @@ -38,144 +38,8 @@ #undef restrict - -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. -#define BLIS_INT_TYPE_SIZE 64 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 64 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS 1 -#define BLIS_NUM_KC_X_NC_BLOCKS 1 -#define BLIS_NUM_MC_X_NC_BLOCKS 1 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. -#define BLIS_SIMD_ALIGN_SIZE 32 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE 32 - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE 64 - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE 64 - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE -#define BLIS_CONTIG_STRIDE_ALIGN_SIZE 32 - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. #define BLIS_BLAS2BLIS_INT_TYPE_SIZE 32 -// Fortran-77 name-mangling macros. -// Underscore is left out to work on BGQ systems -#define PASTEF770(name) name //## _ -#define PASTEF77(ch1,name) ch1 ## name //## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name //## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - #endif diff --git a/config/bulldozer/bli_config.h b/config/bulldozer/bli_config.h index 4b220da1e5..1f99e7e538 100644 --- a/config/bulldozer/bli_config.h +++ b/config/bulldozer/bli_config.h @@ -36,141 +36,7 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. -#define BLIS_INT_TYPE_SIZE 64 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 4 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. -#define BLIS_SIMD_ALIGN_SIZE 16 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. -#define BLIS_BLAS2BLIS_INT_TYPE_SIZE 64 - -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - +#define BLIS_SIMD_ALIGN_SIZE 16 diff --git a/config/carrizo/bli_config.h b/config/carrizo/bli_config.h new file mode 100644 index 0000000000..b0da5de467 --- /dev/null +++ b/config/carrizo/bli_config.h @@ -0,0 +1,48 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef BLIS_CONFIG_H +#define BLIS_CONFIG_H + + +//#define BLIS_ENABLE_PTHREADS +#define BLIS_ENABLE_OPENMP + +#define BLIS_SIMD_ALIGN_SIZE 16 + + + + + +#endif diff --git a/config/carrizo/bli_kernel.h b/config/carrizo/bli_kernel.h new file mode 100644 index 0000000000..cdd1301b84 --- /dev/null +++ b/config/carrizo/bli_kernel.h @@ -0,0 +1,165 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef BLIS_KERNEL_H +#define BLIS_KERNEL_H + + +// -- LEVEL-3 MICRO-KERNEL CONSTANTS ------------------------------------------- + +// -- Cache blocksizes -- + +// +// Constraints: +// +// (1) MC must be a multiple of: +// (a) MR (for zero-padding purposes) +// (b) NR (for zero-padding purposes when MR and NR are "swapped") +// (2) NC must be a multiple of +// (a) NR (for zero-padding purposes) +// (b) MR (for zero-padding purposes when MR and NR are "swapped") +// + +#define BLIS_SGEMM_UKERNEL bli_sgemm_new_16x3 +#define BLIS_DEFAULT_MC_S 528 +#define BLIS_DEFAULT_KC_S 256 +#define BLIS_DEFAULT_NC_S 8400 +#define BLIS_DEFAULT_MR_S 16 +#define BLIS_DEFAULT_NR_S 3 + +#define BLIS_DGEMM_UKERNEL bli_dgemm_new_8x3 +#define BLIS_DEFAULT_MC_D 264 +#define BLIS_DEFAULT_KC_D 256 +#define BLIS_DEFAULT_NC_D 8400 +#define BLIS_DEFAULT_MR_D 8 +#define BLIS_DEFAULT_NR_D 3 + +#define BLIS_CGEMM_UKERNEL bli_cgemm_new_4x2 +#define BLIS_DEFAULT_MC_C 264 +#define BLIS_DEFAULT_KC_C 256 +#define BLIS_DEFAULT_NC_C 8400 +#define BLIS_DEFAULT_MR_C 4 +#define BLIS_DEFAULT_NR_C 2 + +#define BLIS_ZGEMM_UKERNEL bli_zgemm_new_2x2 +#define BLIS_DEFAULT_MC_Z 100 +#define BLIS_DEFAULT_KC_Z 320 +#define BLIS_DEFAULT_NC_Z 8400 +#define BLIS_DEFAULT_MR_Z 2 +#define BLIS_DEFAULT_NR_Z 2 + + + +// -- Register blocksizes -- + + + +// -- Micro-panel alignment -- + + + + + +// -- LEVEL-2 KERNEL CONSTANTS ------------------------------------------------- + + + + +// -- LEVEL-1F KERNEL CONSTANTS ------------------------------------------------ + + + + +// -- LEVEL-3 KERNEL DEFINITIONS ----------------------------------------------- + +// -- gemm -- + + +// -- trsm-related -- + + + + +// -- LEVEL-1M KERNEL DEFINITIONS ---------------------------------------------- + +// -- packm -- + +// -- unpackm -- + + + + +// -- LEVEL-1F KERNEL DEFINITIONS ---------------------------------------------- + +// -- axpy2v -- + +// -- dotaxpyv -- + +// -- axpyf -- + +// -- dotxf -- + +// -- dotxaxpyf -- + + + + +// -- LEVEL-1V KERNEL DEFINITIONS ---------------------------------------------- + +// -- addv -- + +// -- axpyv -- + +// -- copyv -- + +// -- dotv -- + +// -- dotxv -- + +// -- invertv -- + +// -- scal2v -- + +// -- scalv -- + +// -- setv -- + +// -- subv -- + +// -- swapv -- + + + +#endif + diff --git a/config/carrizo/kernels b/config/carrizo/kernels new file mode 120000 index 0000000000..a3f3e222d4 --- /dev/null +++ b/config/carrizo/kernels @@ -0,0 +1 @@ +../../kernels/x86_64/piledriver \ No newline at end of file diff --git a/config/carrizo/make_defs.mk b/config/carrizo/make_defs.mk new file mode 100644 index 0000000000..5f5303ade3 --- /dev/null +++ b/config/carrizo/make_defs.mk @@ -0,0 +1,110 @@ +#!/bin/bash +# +# BLIS +# An object-based framework for developing high-performance BLAS-like +# libraries. +# +# Copyright (C) 2014, The University of Texas at Austin +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# - Neither the name of The University of Texas at Austin nor the names +# of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# Only include this block of code once. +ifndef MAKE_DEFS_MK_INCLUDED +MAKE_DEFS_MK_INCLUDED := yes + + + +# +# --- Build definitions -------------------------------------------------------- +# + +# Variables corresponding to other configure-time options. +BLIS_ENABLE_VERBOSE_MAKE_OUTPUT := no +BLIS_ENABLE_STATIC_BUILD := yes +BLIS_ENABLE_DYNAMIC_BUILD := no + + + +# +# --- Utility program definitions ---------------------------------------------- +# + +SH := /bin/sh +MV := mv +MKDIR := mkdir -p +RM_F := rm -f +RM_RF := rm -rf +SYMLINK := ln -sf +FIND := find +GREP := grep +XARGS := xargs +RANLIB := ranlib +INSTALL := install -c + +# Used to refresh CHANGELOG. +GIT := git +GIT_LOG := $(GIT) log --decorate + + + +# +# --- Development tools definitions -------------------------------------------- +# + +# --- Determine the C compiler and related flags --- +CC := gcc +# Enable IEEE Standard 1003.1-2004 (POSIX.1d). +# NOTE: This is needed to enable posix_memalign(). +CPPROCFLAGS := -D_POSIX_C_SOURCE=200112L +CMISCFLAGS := -std=c99 -fopenmp +CPICFLAGS := -fPIC +CDBGFLAGS := #-g +CWARNFLAGS := -Wall +COPTFLAGS := -O2 -mfpmath=sse -fomit-frame-pointer +CKOPTFLAGS := $(COPTFLAGS) +CVECFLAGS := -mavx -mfma -march=native + +# Aggregate all of the flags into multiple groups: one for standard +# compilation, and one for each of the supported "special" compilation +# modes. +CFLAGS_NOOPT := $(CDBGFLAGS) $(CWARNFLAGS) $(CPICFLAGS) $(CMISCFLAGS) $(CPPROCFLAGS) +CFLAGS := $(COPTFLAGS) $(CVECFLAGS) $(CFLAGS_NOOPT) +CFLAGS_KERNELS := $(CKOPTFLAGS) $(CVECFLAGS) $(CFLAGS_NOOPT) + +# --- Determine the archiver and related flags --- +AR := ar +ARFLAGS := cru + +# --- Determine the linker and related flags --- +LINKER := $(CC) +SOFLAGS := -shared +LDFLAGS := -lm -fopenmp + + + +# end of ifndef MAKE_DEFS_MK_INCLUDED conditional block +endif diff --git a/config/cortex-a15/bli_config.h b/config/cortex-a15/bli_config.h index a05f8d0827..7537125400 100644 --- a/config/cortex-a15/bli_config.h +++ b/config/cortex-a15/bli_config.h @@ -36,142 +36,9 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. #define BLIS_INT_TYPE_SIZE 32 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 1 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. #define BLIS_SIMD_ALIGN_SIZE 16 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. #define BLIS_BLAS2BLIS_INT_TYPE_SIZE 32 -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - - #endif diff --git a/config/cortex-a9/bli_config.h b/config/cortex-a9/bli_config.h index a05f8d0827..7537125400 100644 --- a/config/cortex-a9/bli_config.h +++ b/config/cortex-a9/bli_config.h @@ -36,142 +36,9 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. #define BLIS_INT_TYPE_SIZE 32 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 1 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. #define BLIS_SIMD_ALIGN_SIZE 16 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. #define BLIS_BLAS2BLIS_INT_TYPE_SIZE 32 -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - - #endif diff --git a/config/dunnington/bli_config.h b/config/dunnington/bli_config.h index 13cb2c1f64..64392de999 100644 --- a/config/dunnington/bli_config.h +++ b/config/dunnington/bli_config.h @@ -36,142 +36,7 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. -#define BLIS_INT_TYPE_SIZE 64 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 1 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS 1 -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. -#define BLIS_SIMD_ALIGN_SIZE 16 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE 16 - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE 16 - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. -#define BLIS_BLAS2BLIS_INT_TYPE_SIZE 64 - -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - +#define BLIS_SIMD_ALIGN_SIZE 16 #endif diff --git a/config/emscripten/bli_config.h b/config/emscripten/bli_config.h index a05f8d0827..7537125400 100644 --- a/config/emscripten/bli_config.h +++ b/config/emscripten/bli_config.h @@ -36,142 +36,9 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. #define BLIS_INT_TYPE_SIZE 32 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 1 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. #define BLIS_SIMD_ALIGN_SIZE 16 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. #define BLIS_BLAS2BLIS_INT_TYPE_SIZE 32 -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - - #endif diff --git a/config/haswell/bli_config.h b/config/haswell/bli_config.h new file mode 100644 index 0000000000..5f66f6dae4 --- /dev/null +++ b/config/haswell/bli_config.h @@ -0,0 +1,46 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef BLIS_CONFIG_H +#define BLIS_CONFIG_H + +// Enable multithreading via POSIX threads. +//#define BLIS_ENABLE_PTHREADS + +// Enable multithreading via OpenMP. +#define BLIS_ENABLE_OPENMP + + + +#endif diff --git a/config/haswell/bli_kernel.h b/config/haswell/bli_kernel.h new file mode 100644 index 0000000000..9df503005e --- /dev/null +++ b/config/haswell/bli_kernel.h @@ -0,0 +1,172 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef BLIS_KERNEL_H +#define BLIS_KERNEL_H + + +// -- LEVEL-3 MICRO-KERNEL CONSTANTS AND DEFINITIONS --------------------------- + +// +// Constraints: +// +// (1) MC must be a multiple of: +// (a) MR (for zero-padding purposes) +// (b) NR (for zero-padding purposes when MR and NR are "swapped") +// (2) NC must be a multiple of +// (a) NR (for zero-padding purposes) +// (b) MR (for zero-padding purposes when MR and NR are "swapped") +// + +#if 0 + +#define BLIS_SGEMM_UKERNEL bli_sgemm_asm_24x4 +#define BLIS_DEFAULT_MC_S 264 +#define BLIS_DEFAULT_KC_S 128 +#define BLIS_DEFAULT_NC_S 4080 +#define BLIS_DEFAULT_MR_S 24 +#define BLIS_DEFAULT_NR_S 4 + +#else + +#define BLIS_SGEMM_UKERNEL bli_sgemm_asm_16x6 +#define BLIS_DEFAULT_MC_S 144 +#define BLIS_DEFAULT_KC_S 256 +#define BLIS_DEFAULT_NC_S 4080 +#define BLIS_DEFAULT_MR_S 16 +#define BLIS_DEFAULT_NR_S 6 + +#endif + +#if 0 + +#define BLIS_DGEMM_UKERNEL bli_dgemm_asm_12x4 +#define BLIS_DEFAULT_MC_D 96 +#define BLIS_DEFAULT_KC_D 192 +#define BLIS_DEFAULT_NC_D 4080 +#define BLIS_DEFAULT_MR_D 12 +#define BLIS_DEFAULT_NR_D 4 + +#else + +#define BLIS_DGEMM_UKERNEL bli_dgemm_asm_8x6 +#define BLIS_DEFAULT_MC_D 72 +#define BLIS_DEFAULT_KC_D 256 +#define BLIS_DEFAULT_NC_D 4080 +#define BLIS_DEFAULT_MR_D 8 +#define BLIS_DEFAULT_NR_D 6 + +#endif + +/* +#define BLIS_CGEMM_UKERNEL bli_cgemm_asm_8x4 +#define BLIS_DEFAULT_MC_C 96 +#define BLIS_DEFAULT_KC_C 256 +#define BLIS_DEFAULT_NC_C 4096 +#define BLIS_DEFAULT_MR_C 8 +#define BLIS_DEFAULT_NR_C 4 + +#define BLIS_ZGEMM_UKERNEL bli_zgemm_asm_4x4 +#define BLIS_DEFAULT_MC_Z 64 +#define BLIS_DEFAULT_KC_Z 192 +#define BLIS_DEFAULT_NC_Z 4096 +#define BLIS_DEFAULT_MR_Z 4 +#define BLIS_DEFAULT_NR_Z 4 +*/ + + + + +// -- LEVEL-2 KERNEL CONSTANTS ------------------------------------------------- + + + + +// -- LEVEL-1F KERNEL CONSTANTS ------------------------------------------------ + + + + +// -- LEVEL-1M KERNEL DEFINITIONS ---------------------------------------------- + +// -- packm -- + +// -- unpackm -- + + + + +// -- LEVEL-1F KERNEL DEFINITIONS ---------------------------------------------- + +// -- axpy2v -- + +// -- dotaxpyv -- + +// -- axpyf -- + +// -- dotxf -- + +// -- dotxaxpyf -- + + + + +// -- LEVEL-1V KERNEL DEFINITIONS ---------------------------------------------- + +// -- addv -- + +// -- axpyv -- + +// -- copyv -- + +// -- dotv -- + +// -- dotxv -- + +// -- invertv -- + +// -- scal2v -- + +// -- scalv -- + +// -- setv -- + +// -- subv -- + +// -- swapv -- + + + +#endif + diff --git a/config/haswell/kernels b/config/haswell/kernels new file mode 120000 index 0000000000..58026848ba --- /dev/null +++ b/config/haswell/kernels @@ -0,0 +1 @@ +../../kernels/x86_64/avx2 \ No newline at end of file diff --git a/config/haswell/make_defs.mk b/config/haswell/make_defs.mk new file mode 100644 index 0000000000..a854c47c57 --- /dev/null +++ b/config/haswell/make_defs.mk @@ -0,0 +1,110 @@ +#!/bin/bash +# +# BLIS +# An object-based framework for developing high-performance BLAS-like +# libraries. +# +# Copyright (C) 2014, The University of Texas at Austin +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# - Neither the name of The University of Texas at Austin nor the names +# of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# Only include this block of code once. +ifndef MAKE_DEFS_MK_INCLUDED +MAKE_DEFS_MK_INCLUDED := yes + + + +# +# --- Build definitions -------------------------------------------------------- +# + +# Variables corresponding to other configure-time options. +BLIS_ENABLE_VERBOSE_MAKE_OUTPUT := no +BLIS_ENABLE_STATIC_BUILD := yes +BLIS_ENABLE_DYNAMIC_BUILD := no + + + +# +# --- Utility program definitions ---------------------------------------------- +# + +SH := /bin/sh +MV := mv +MKDIR := mkdir -p +RM_F := rm -f +RM_RF := rm -rf +SYMLINK := ln -sf +FIND := find +GREP := grep +XARGS := xargs +RANLIB := ranlib +INSTALL := install -c + +# Used to refresh CHANGELOG. +GIT := git +GIT_LOG := $(GIT) log --decorate + + + +# +# --- Development tools definitions -------------------------------------------- +# + +# --- Determine the C compiler and related flags --- +CC := gcc +# Enable IEEE Standard 1003.1-2004 (POSIX.1d). +# NOTE: This is needed to enable posix_memalign(). +CPPROCFLAGS := -D_POSIX_C_SOURCE=200112L +CMISCFLAGS := -std=c99 -m64 -fopenmp # -fopenmp -pg +CPICFLAGS := -fPIC +CDBGFLAGS := #-g +CWARNFLAGS := -Wall +COPTFLAGS := -O3 -march=native +CKOPTFLAGS := $(COPTFLAGS) +CVECFLAGS := -mavx2 -mfma -mfpmath=sse #-msse3 -march=native # -mfpmath=sse + +# Aggregate all of the flags into multiple groups: one for standard +# compilation, and one for each of the supported "special" compilation +# modes. +CFLAGS_NOOPT := $(CDBGFLAGS) $(CWARNFLAGS) $(CPICFLAGS) $(CMISCFLAGS) $(CPPROCFLAGS) +CFLAGS := $(COPTFLAGS) $(CVECFLAGS) $(CFLAGS_NOOPT) +CFLAGS_KERNELS := $(CKOPTFLAGS) $(CVECFLAGS) $(CFLAGS_NOOPT) + +# --- Determine the archiver and related flags --- +AR := ar +ARFLAGS := cru + +# --- Determine the linker and related flags --- +LINKER := $(CC) +SOFLAGS := -shared +LDFLAGS := -lm -fopenmp -lpthread + + + +# end of ifndef MAKE_DEFS_MK_INCLUDED conditional block +endif diff --git a/config/loongson3a/bli_config.h b/config/loongson3a/bli_config.h index 42ae3a4bf1..7537125400 100644 --- a/config/loongson3a/bli_config.h +++ b/config/loongson3a/bli_config.h @@ -36,142 +36,9 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. #define BLIS_INT_TYPE_SIZE 32 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 8 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS 1 -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. #define BLIS_SIMD_ALIGN_SIZE 16 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE 16 - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE 16 - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE 16 - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. #define BLIS_BLAS2BLIS_INT_TYPE_SIZE 32 -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - - #endif diff --git a/config/mic/bli_config.h b/config/mic/bli_config.h index 8d11e5da80..a119a2dde1 100644 --- a/config/mic/bli_config.h +++ b/config/mic/bli_config.h @@ -39,146 +39,10 @@ #define BLIS_TREE_BARRIER #define BLIS_TREE_BARRIER_ARITY 4 -#define BLIS_DGEMM_UKERNEL_PREFERS_CONTIG_ROWS -#define BLIS_ENABLE_MULTITHREADING #define BLIS_ENABLE_OPENMP -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. -#define BLIS_INT_TYPE_SIZE 64 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 60 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS 1 -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 256 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. #define BLIS_SIMD_ALIGN_SIZE 32 -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. -#define BLIS_BLAS2BLIS_INT_TYPE_SIZE 64 - -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - #endif diff --git a/config/mic/bli_kernel.h b/config/mic/bli_kernel.h index 8e86791223..880e97d358 100644 --- a/config/mic/bli_kernel.h +++ b/config/mic/bli_kernel.h @@ -151,6 +151,8 @@ // -- gemm -- +#define BLIS_DGEMM_UKERNEL_PREFERS_CONTIG_ROWS + #define BLIS_DGEMM_UKERNEL bli_dgemm_opt_30x8 #define BLIS_SGEMM_UKERNEL bli_sgemm_opt_30x16 diff --git a/config/piledriver/bli_config.h b/config/piledriver/bli_config.h index 6bd2262b50..dce91516da 100644 --- a/config/piledriver/bli_config.h +++ b/config/piledriver/bli_config.h @@ -36,148 +36,11 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. -#define BLIS_INT_TYPE_SIZE 64 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 4 - -// Enable multithreading via POSIX threads. //#define BLIS_ENABLE_PTHREADS -// Enable multithreading via OpenMP. #define BLIS_ENABLE_OPENMP - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. -#define BLIS_SIMD_ALIGN_SIZE 16 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. -#define BLIS_BLAS2BLIS_INT_TYPE_SIZE 64 - -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - +#define BLIS_SIMD_ALIGN_SIZE 16 #endif diff --git a/config/pnacl/bli_config.h b/config/pnacl/bli_config.h index a05f8d0827..7537125400 100644 --- a/config/pnacl/bli_config.h +++ b/config/pnacl/bli_config.h @@ -36,142 +36,9 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. #define BLIS_INT_TYPE_SIZE 32 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 1 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. #define BLIS_SIMD_ALIGN_SIZE 16 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. #define BLIS_BLAS2BLIS_INT_TYPE_SIZE 32 -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - - #endif diff --git a/config/power7/bli_config.h b/config/power7/bli_config.h index 5a3ce1e1bd..e1725b9b05 100644 --- a/config/power7/bli_config.h +++ b/config/power7/bli_config.h @@ -36,139 +36,6 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. -#define BLIS_INT_TYPE_SIZE 64 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 24 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS 1 -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. -#define BLIS_SIMD_ALIGN_SIZE 16 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE 16 - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE 16 - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. -#define BLIS_BLAS2BLIS_INT_TYPE_SIZE 64 - -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - diff --git a/config/reference/bli_config.h b/config/reference/bli_config.h index 65463146c7..5195e61c5b 100644 --- a/config/reference/bli_config.h +++ b/config/reference/bli_config.h @@ -36,143 +36,6 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. -#define BLIS_INT_TYPE_SIZE 32 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 1 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. -#define BLIS_SIMD_ALIGN_SIZE 16 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. -#define BLIS_BLAS2BLIS_INT_TYPE_SIZE 32 - -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - - #endif diff --git a/config/sandybridge/bli_config.h b/config/sandybridge/bli_config.h index 1d2a4d6bb2..5f66f6dae4 100644 --- a/config/sandybridge/bli_config.h +++ b/config/sandybridge/bli_config.h @@ -35,42 +35,6 @@ #ifndef BLIS_CONFIG_H #define BLIS_CONFIG_H - -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. -#define BLIS_INT_TYPE_SIZE 64 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 16 - // Enable multithreading via POSIX threads. //#define BLIS_ENABLE_PTHREADS @@ -79,105 +43,4 @@ -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS 2 -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. -#define BLIS_SIMD_ALIGN_SIZE 32 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. -#define BLIS_BLAS2BLIS_INT_TYPE_SIZE 64 - -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - - - #endif diff --git a/config/template/bli_config.h b/config/template/bli_config.h index fc65e5135f..cc8e64b5a2 100644 --- a/config/template/bli_config.h +++ b/config/template/bli_config.h @@ -36,142 +36,6 @@ #define BLIS_CONFIG_H -// -- OPERATING SYSTEM --------------------------------------------------------- - - - -// -- INTEGER PROPERTIES ------------------------------------------------------- - -// The bit size of the integer type used to track values such as dimensions, -// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed -// integers while 64 results in 64-bit integers. Any other value results in use -// of the C99 type "long int". Note that this ONLY affects integers used -// internally within BLIS as well as those exposed in the native BLAS-like BLIS -// interface. -#define BLIS_INT_TYPE_SIZE 64 - - - -// -- FLOATING-POINT PROPERTIES ------------------------------------------------ - -// Define the number of floating-point types supported, and the size of the -// largest type. -#define BLIS_NUM_FP_TYPES 4 -#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) - -// Enable use of built-in C99 "float complex" and "double complex" types and -// associated overloaded operations and functions? Disabling results in -// scomplex and dcomplex being defined in terms of simple structs. -//#define BLIS_ENABLE_C99_COMPLEX - - - -// -- MULTITHREADING ----------------------------------------------------------- - -// The maximum number of BLIS threads that will run concurrently. -#define BLIS_MAX_NUM_THREADS 1 - - - -// -- MEMORY ALLOCATION -------------------------------------------------------- - -// -- Contiguous (static) memory allocator -- - -// The number of MC x KC, KC x NC, and MC x NC blocks to reserve in the -// contiguous memory pools. -#define BLIS_NUM_MC_X_KC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_KC_X_NC_BLOCKS BLIS_MAX_NUM_THREADS -#define BLIS_NUM_MC_X_NC_BLOCKS 0 - -// The maximum preload byte offset is used to pad the end of the contiguous -// memory pools so that the micro-kernel, when computing with the end of the -// last block, can exceed the bounds of the usable portion of the memory -// region without causing a segmentation fault. -#define BLIS_MAX_PRELOAD_BYTE_OFFSET 128 - -// -- Memory alignment -- - -// It is sometimes useful to define the various memory alignments in terms -// of some other characteristics of the system, such as the cache line size -// and the page size. -#define BLIS_CACHE_LINE_SIZE 64 -#define BLIS_PAGE_SIZE 4096 - -// Alignment size needed by the instruction set for aligned SIMD/vector -// instructions. -#define BLIS_SIMD_ALIGN_SIZE 16 - -// Alignment size used to align local stack buffers within macro-kernel -// functions. -#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when allocating memory dynamically from the operating -// system (eg: posix_memalign()). To disable heap alignment and just use -// malloc() instead, set this to 1. -#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE - -// Alignment size used when sizing leading dimensions of dynamically -// allocated memory. -#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_CACHE_LINE_SIZE - -// Alignment size used when allocating entire blocks of contiguous memory -// from the contiguous memory allocator. -#define BLIS_CONTIG_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE - - - -// -- MIXED DATATYPE SUPPORT --------------------------------------------------- - -// Basic (homogeneous) datatype support always enabled. - -// Enable mixed domain operations? -//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT - -// Enable extra mixed precision operations? -//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT - - - -// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- - -// Stay initialized after auto-initialization, unless and until the user -// explicitly calls bli_finalize(). -#define BLIS_ENABLE_STAY_AUTO_INITIALIZED - - - -// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- - -// Enable the BLAS compatibility layer? -#define BLIS_ENABLE_BLAS2BLIS - -// The bit size of the integer type used to track values such as dimensions and -// leading dimensions (ie: column strides) within the BLAS compatibility layer. -// A value of 32 results in the compatibility layer using 32-bit signed integers -// while 64 results in 64-bit integers. Any other value results in use of the -// C99 type "long int". Note that this ONLY affects integers used within the -// BLAS compatibility layer. -#define BLIS_BLAS2BLIS_INT_TYPE_SIZE 64 - -// Fortran-77 name-mangling macros. -#define PASTEF770(name) name ## _ -#define PASTEF77(ch1,name) ch1 ## name ## _ -#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ -#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ - - - -// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ - -// Enable the CBLAS compatibility layer? -// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer -// regardless of whether or not it was explicitly enabled above. Furthermore, -// the CBLAS compatibility layer will use the integer type size definition -// specified above when defining the size of its own integers (regardless of -// whether the BLAS layer was enabled directly or indirectly). -//#define BLIS_ENABLE_CBLAS - - - #endif + diff --git a/configure b/configure index 3c8dbbd278..aa36fa7868 100755 --- a/configure +++ b/configure @@ -145,6 +145,9 @@ main() # Option variables. quiet_flag='' + # The path to the auto-detection script. + auto_detect_sh="${build_dirpath}/auto-detect/auto-detect.sh" + # The name of the reference configuration sub-directory. config_name_ref='reference' @@ -216,11 +219,18 @@ main() elif [ $# = "1" ]; then - if [ $1 = "auto" ]; then - config_name=`${build_dirpath}/auto-detect/auto-detect.sh` - else - config_name=$1 + if [ $1 = "auto" ]; then + + echo "${script_name}: automatic configuration requested." + config_name=`${build_dirpath}/auto-detect/auto-detect.sh` + #config_name=$(${auto_detect_sh}) + + else + + echo "${script_name}: manual configuration requested." + config_name=$1 fi + echo "${script_name}: configuring with '${config_name}' configuration sub-directory." else diff --git a/frame/1m/packm/bli_packm.h b/frame/1m/packm/bli_packm.h index c2d6b40766..26030bb2d9 100644 --- a/frame/1m/packm/bli_packm.h +++ b/frame/1m/packm/bli_packm.h @@ -42,7 +42,6 @@ #include "bli_packm_unb_var1.h" #include "bli_packm_blk_var1.h" -#include "bli_packm_blk_var2.h" #include "bli_packm_struc_cxk.h" #include "bli_packm_struc_cxk_4mi.h" diff --git a/frame/1m/packm/bli_packm_blk_var1.c b/frame/1m/packm/bli_packm_blk_var1.c index 78d52c9cad..3c0318bad8 100644 --- a/frame/1m/packm/bli_packm_blk_var1.c +++ b/frame/1m/packm/bli_packm_blk_var1.c @@ -62,6 +62,9 @@ typedef void (*FUNCPTR_T)( static FUNCPTR_T GENARRAY(ftypes,packm_blk_var1); extern func_t* packm_struc_cxk_kers; +extern func_t* packm_struc_cxk_4mi_kers; +extern func_t* packm_struc_cxk_3mis_kers; +extern func_t* packm_struc_cxk_rih_kers; void bli_packm_blk_var1( obj_t* c, @@ -96,6 +99,8 @@ void bli_packm_blk_var1( obj_t* c, dim_t pd_p = bli_obj_panel_dim( *p ); inc_t ps_p = bli_obj_panel_stride( *p ); + obj_t kappa; + obj_t* kappa_p; void* buf_kappa; func_t* packm_kers; @@ -103,14 +108,61 @@ void bli_packm_blk_var1( obj_t* c, FUNCPTR_T f; - // This variant assumes that the micro-kernel will always apply the - // alpha scalar of the higher-level operation. Thus, we use BLIS_ONE - // for kappa so that the underlying packm implementation does not - // scale during packing. - buf_kappa = bli_obj_buffer_for_const( dt_cp, BLIS_ONE ); + // Treatment of kappa (ie: packing during scaling) depends on + // whether we are executing an induced method. + if ( bli_is_ind_packed( schema ) ) + { + // The value for kappa we use will depend on whether the scalar + // attached to A has a nonzero imaginary component. If it does, + // then we will apply the scalar during packing to facilitate + // implementing induced complex domain algorithms in terms of + // real domain micro-kernels. (In the aforementioned situation, + // applying a real scalar is easy, but applying a complex one is + // harder, so we avoid the need altogether with the code below.) + if( thread_am_ochief( t ) ) + { + if ( bli_obj_scalar_has_nonzero_imag( p ) ) + { + // Detach the scalar. + bli_obj_scalar_detach( p, &kappa ); + + // Reset the attached scalar (to 1.0). + bli_obj_scalar_reset( p ); + + kappa_p = κ + } + else + { + // If the internal scalar of A has only a real component, then + // we will apply it later (in the micro-kernel), and so we will + // use BLIS_ONE to indicate no scaling during packing. + kappa_p = &BLIS_ONE; + } + } + kappa_p = thread_obroadcast( t, kappa_p ); + + // Acquire the buffer to the kappa chosen above. + buf_kappa = bli_obj_buffer_for_1x1( dt_cp, *kappa_p ); + } + else // if ( bli_is_nat_packed( schema ) ) + { + // This branch if for native execution, where we assume that + // the micro-kernel will always apply the alpha scalar of the + // higher-level operation. Thus, we use BLIS_ONE for kappa so + // that the underlying packm implementation does not perform + // any scaling during packing. + buf_kappa = bli_obj_buffer_for_const( dt_cp, BLIS_ONE ); + } - // Choose the correct func_t object. - packm_kers = packm_struc_cxk_kers; + + // Choose the correct func_t object based on the pack_t schema. + if ( bli_is_4mi_packed( schema ) ) packm_kers = packm_struc_cxk_4mi_kers; + else if ( bli_is_3mi_packed( schema ) || + bli_is_3ms_packed( schema ) ) packm_kers = packm_struc_cxk_3mis_kers; + else if ( bli_is_ro_packed( schema ) || + bli_is_io_packed( schema ) || + bli_is_rpi_packed( schema ) ) packm_kers = packm_struc_cxk_rih_kers; + else packm_kers = packm_struc_cxk_kers; // Query the datatype-specific function pointer from the func_t object. packm_ker = bli_func_obj_query( dt_cp, packm_kers ); @@ -144,8 +196,8 @@ void bli_packm_blk_var1( obj_t* c, } -#undef GENTFUNC -#define GENTFUNC( ctype, ch, varname, kertype ) \ +#undef GENTFUNCR +#define GENTFUNCR( ctype, ctype_r, ch, chr, varname, kertype ) \ \ void PASTEMAC(ch,varname)( \ struc_t strucc, \ @@ -204,6 +256,9 @@ void PASTEMAC(ch,varname)( \ conj_t conjc; \ bool_t row_stored; \ bool_t col_stored; \ + inc_t is_p_use; \ + dim_t ss_num; \ + dim_t ss_den; \ \ ctype* restrict c_use; \ ctype* restrict p_use; \ @@ -274,6 +329,17 @@ void PASTEMAC(ch,varname)( \ m_panel_max = &panel_dim_max; \ n_panel_max = &panel_len_max_i; \ } \ +\ + /* Compute the storage stride scaling. Usually this is just 1. However, + in the case of interleaved 3m, we need to scale by 3/2, and in the + cases of real-only, imag-only, or summed-only, we need to scale by + 1/2. In both cases, we are compensating for the fact that pointer + arithmetic occurs in terms of complex elements rather than real + elements. */ \ + if ( bli_is_3mi_packed( schema ) ) { ss_num = 3; ss_den = 2; } \ + else if ( bli_is_3ms_packed( schema ) ) { ss_num = 1; ss_den = 2; } \ + else if ( bli_is_rih_packed( schema ) ) { ss_num = 1; ss_den = 2; } \ + else { ss_num = 1; ss_den = 1; } \ \ /* Compute the total number of iterations we'll need. */ \ num_iter = iter_dim / panel_dim_max + ( iter_dim % panel_dim_max ? 1 : 0 ); \ @@ -297,6 +363,15 @@ void PASTEMAC(ch,varname)( \ } \ \ p_begin = p_cast; \ +\ +/* +if ( row_stored ) \ +PASTEMAC(ch,fprintm)( stdout, "packm_var2: b", m, n, \ + c_cast, rs_c, cs_c, "%4.1f", "" ); \ +if ( col_stored ) \ +PASTEMAC(ch,fprintm)( stdout, "packm_var2: a", m, n, \ + c_cast, rs_c, cs_c, "%4.1f", "" ); \ +*/ \ \ for ( ic = ic0, ip = ip0, it = 0; it < num_iter; \ ic += ic_inc, ip += ip_inc, it += 1 ) \ @@ -353,6 +428,15 @@ void PASTEMAC(ch,varname)( \ \ c_use = c_begin + (panel_off_i )*ldc; \ p_use = p_begin; \ +\ + /* We need to re-compute the imaginary stride as a function of + panel_len_max_i since triangular packed matrices have panels + of varying lengths. NOTE: This imaginary stride value is + only referenced by the packm kernels for induced methods. */ \ + is_p_use = ldp * panel_len_max_i; \ +\ + /* We nudge the imaginary stride up by one if it is odd. */ \ + is_p_use += ( bli_is_odd( is_p_use ) ? 1 : 0 ); \ \ if( packm_thread_my_iter( it, thread ) ) \ { \ @@ -370,25 +454,27 @@ void PASTEMAC(ch,varname)( \ kappa_cast, \ c_use, rs_c, cs_c, \ p_use, rs_p, cs_p, \ - is_p ); \ + is_p_use ); \ } \ \ /* NOTE: This value is usually LESS than ps_p because triangular matrices usually have several micro-panels that are shorter than a "full" micro-panel. */ \ - p_inc = ldp * panel_len_max_i; \ -\ - /* We nudge the panel increment up by one if it is odd. */ \ - p_inc += ( bli_is_odd( p_inc ) ? 1 : 0 ); \ + p_inc = ( is_p_use * ss_num ) / ss_den; \ } \ else if ( bli_is_herm_or_symm( strucc ) ) \ { \ /* This case executes if the panel belongs to a Hermitian or symmetric matrix, which includes stored, unstored, and diagonal-intersecting panels. */ \ +\ + c_use = c_begin; \ + p_use = p_begin; \ \ panel_len_i = panel_len_full; \ panel_len_max_i = panel_len_max; \ +\ + is_p_use = is_p; \ \ if( packm_thread_my_iter( it, thread ) ) \ { \ @@ -404,13 +490,11 @@ void PASTEMAC(ch,varname)( \ *m_panel_max, \ *n_panel_max, \ kappa_cast, \ - c_begin, rs_c, cs_c, \ - p_begin, rs_p, cs_p, \ - is_p ); \ + c_use, rs_c, cs_c, \ + p_use, rs_p, cs_p, \ + is_p_use ); \ } \ \ - /* NOTE: This value is equivalent to ps_p. */ \ - /*p_inc = ldp * panel_len_max_i;*/ \ p_inc = ps_p; \ } \ else \ @@ -418,9 +502,14 @@ void PASTEMAC(ch,varname)( \ /* This case executes if the panel is general, or, if the panel is part of a triangular matrix and is neither unstored (ie: zero) nor diagonal-intersecting. */ \ +\ + c_use = c_begin; \ + p_use = p_begin; \ \ panel_len_i = panel_len_full; \ panel_len_max_i = panel_len_max; \ +\ + is_p_use = is_p; \ \ if( packm_thread_my_iter( it, thread ) ) \ { \ @@ -436,28 +525,81 @@ void PASTEMAC(ch,varname)( \ *m_panel_max, \ *n_panel_max, \ kappa_cast, \ - c_begin, rs_c, cs_c, \ - p_begin, rs_p, cs_p, \ - is_p ); \ + c_use, rs_c, cs_c, \ + p_use, rs_p, cs_p, \ + is_p_use ); \ } \ -/* - if ( row_stored ) \ - PASTEMAC(ch,fprintm)( stdout, "packm_var1: bp copied", panel_len_max_i, panel_dim_max, \ - p_begin, rs_p, cs_p, "%9.2e", "" ); \ - else if ( col_stored ) \ - PASTEMAC(ch,fprintm)( stdout, "packm_var1: ap copied", panel_dim_max, panel_len_max_i, \ - p_begin, rs_p, cs_p, "%9.2e", "" ); \ -*/ \ \ /* NOTE: This value is equivalent to ps_p. */ \ - /*p_inc = ldp * panel_len_max_i;*/ \ p_inc = ps_p; \ } \ \ +/* + if ( bli_is_4mi_packed( schema ) ) { \ + printf( "packm_var2: is_p_use = %lu\n", is_p_use ); \ + if ( col_stored ) { \ + if ( 0 ) \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: a_r", *m_panel_use, *n_panel_use, \ + ( ctype_r* )c_use, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: ap_r", *m_panel_max, *n_panel_max, \ + ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: ap_i", *m_panel_max, *n_panel_max, \ + ( ctype_r* )p_use + is_p_use, rs_p, cs_p, "%4.1f", "" ); \ + } \ + if ( row_stored ) { \ + if ( 0 ) \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: b_r", *m_panel_use, *n_panel_use, \ + ( ctype_r* )c_use, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: bp_r", *m_panel_max, *n_panel_max, \ + ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: bp_i", *m_panel_max, *n_panel_max, \ + ( ctype_r* )p_use + is_p_use, rs_p, cs_p, "%4.1f", "" ); \ + } \ + } \ +*/ \ +/* +*/ \ +\ +/* +*/ \ +/* + PASTEMAC(chr,fprintm)( stdout, "packm_var2: bp_rpi", *m_panel_max, *n_panel_max, \ + ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ +*/ \ +\ +\ +/* + if ( row_stored ) { \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: b_r", *m_panel_max, *n_panel_max, \ + ( ctype_r* )c_use, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: b_i", *m_panel_max, *n_panel_max, \ + (( ctype_r* )c_use)+rs_c, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: bp_r", *m_panel_max, *n_panel_max, \ + ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ + inc_t is_b = rs_p * *m_panel_max; \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: bp_i", *m_panel_max, *n_panel_max, \ + ( ctype_r* )p_use + is_b, rs_p, cs_p, "%4.1f", "" ); \ + } \ +*/ \ +\ +\ +/* + if ( col_stored ) { \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: a_r", *m_panel_max, *n_panel_max, \ + ( ctype_r* )c_use, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: a_i", *m_panel_max, *n_panel_max, \ + (( ctype_r* )c_use)+rs_c, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: ap_r", *m_panel_max, *n_panel_max, \ + ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ + PASTEMAC(chr,fprintm)( stdout, "packm_var2: ap_i", *m_panel_max, *n_panel_max, \ + ( ctype_r* )p_use + p_inc, rs_p, cs_p, "%4.1f", "" ); \ + } \ +*/ \ \ p_begin += p_inc; \ +\ } \ } -INSERT_GENTFUNC_BASIC( packm_blk_var1, packm_ker_t ) +INSERT_GENTFUNCR_BASIC( packm_blk_var1, packm_ker_t ) diff --git a/frame/1m/packm/bli_packm_blk_var2.c b/frame/1m/packm/bli_packm_blk_var1.c.old similarity index 70% rename from frame/1m/packm/bli_packm_blk_var2.c rename to frame/1m/packm/bli_packm_blk_var1.c.old index e8c700bb7a..78d52c9cad 100644 --- a/frame/1m/packm/bli_packm_blk_var2.c +++ b/frame/1m/packm/bli_packm_blk_var1.c.old @@ -59,15 +59,12 @@ typedef void (*FUNCPTR_T)( packm_thrinfo_t* thread ); -//static FUNCPTR_T GENARRAY(ftypes,packm_blk_var2); +static FUNCPTR_T GENARRAY(ftypes,packm_blk_var1); extern func_t* packm_struc_cxk_kers; -extern func_t* packm_struc_cxk_4mi_kers; -extern func_t* packm_struc_cxk_3mis_kers; -extern func_t* packm_struc_cxk_rih_kers; -void bli_packm_blk_var2( obj_t* c, +void bli_packm_blk_var1( obj_t* c, obj_t* p, packm_thrinfo_t* t ) { @@ -99,8 +96,6 @@ void bli_packm_blk_var2( obj_t* c, dim_t pd_p = bli_obj_panel_dim( *p ); inc_t ps_p = bli_obj_panel_stride( *p ); - obj_t kappa; - obj_t* kappa_p; void* buf_kappa; func_t* packm_kers; @@ -108,59 +103,14 @@ void bli_packm_blk_var2( obj_t* c, FUNCPTR_T f; + // This variant assumes that the micro-kernel will always apply the + // alpha scalar of the higher-level operation. Thus, we use BLIS_ONE + // for kappa so that the underlying packm implementation does not + // scale during packing. + buf_kappa = bli_obj_buffer_for_const( dt_cp, BLIS_ONE ); -/* - // We want this variant to behave identically to that of variant 1 - // in the real domain. - if ( bli_is_real( dt_cp ) ) - { - bli_packm_blk_var1( c, p, t ); - return; - } -*/ - - // The value for kappa we use will depend on whether the scalar - // attached to A has a nonzero imaginary component. If it does, - // then we will apply the scalar during packing to facilitate - // implementing induced complex domain algorithms in terms of - // real domain micro-kernels. (In the aforementioned situation, - // applying a real scalar is easy, but applying a complex one is - // harder, so we avoid the need altogether with the code below.) - if( thread_am_ochief( t ) ) - { - if ( bli_obj_scalar_has_nonzero_imag( p ) ) - { - // Detach the scalar. - bli_obj_scalar_detach( p, &kappa ); - - // Reset the attached scalar (to 1.0). - bli_obj_scalar_reset( p ); - - kappa_p = κ - } - else - { - // If the internal scalar of A has only a real component, then - // we will apply it later (in the micro-kernel), and so we will - // use BLIS_ONE to indicate no scaling during packing. - kappa_p = &BLIS_ONE; - } - } - kappa_p = thread_obroadcast( t, kappa_p ); - - - // Acquire the buffer to the kappa chosen above. - buf_kappa = bli_obj_buffer_for_1x1( dt_cp, *kappa_p ); - - - // Choose the correct func_t object based on the pack_t schema. - if ( bli_is_4mi_packed( schema ) ) packm_kers = packm_struc_cxk_4mi_kers; - else if ( bli_is_3mi_packed( schema ) || - bli_is_3ms_packed( schema ) ) packm_kers = packm_struc_cxk_3mis_kers; - else if ( bli_is_ro_packed( schema ) || - bli_is_io_packed( schema ) || - bli_is_rpi_packed( schema ) ) packm_kers = packm_struc_cxk_rih_kers; - else packm_kers = packm_struc_cxk_kers; + // Choose the correct func_t object. + packm_kers = packm_struc_cxk_kers; // Query the datatype-specific function pointer from the func_t object. packm_ker = bli_func_obj_query( dt_cp, packm_kers ); @@ -168,9 +118,7 @@ void bli_packm_blk_var2( obj_t* c, // Index into the type combination array to extract the correct // function pointer. - //f = ftypes[dt_cp]; - if ( bli_is_scomplex( dt_cp ) ) f = bli_cpackm_blk_var2; - else f = bli_zpackm_blk_var2; + f = ftypes[dt_cp]; // Invoke the function. f( strucc, @@ -196,8 +144,8 @@ void bli_packm_blk_var2( obj_t* c, } -#undef GENTFUNCCO -#define GENTFUNCCO( ctype, ctype_r, ch, chr, varname, kertype ) \ +#undef GENTFUNC +#define GENTFUNC( ctype, ch, varname, kertype ) \ \ void PASTEMAC(ch,varname)( \ struc_t strucc, \ @@ -256,8 +204,6 @@ void PASTEMAC(ch,varname)( \ conj_t conjc; \ bool_t row_stored; \ bool_t col_stored; \ - dim_t ss_num; \ - dim_t ss_den; \ \ ctype* restrict c_use; \ ctype* restrict p_use; \ @@ -328,17 +274,6 @@ void PASTEMAC(ch,varname)( \ m_panel_max = &panel_dim_max; \ n_panel_max = &panel_len_max_i; \ } \ -\ - /* Compute the storage stride scaling. Usually this is just 1. However, - in the case of interleaved 3m, we need to scale by 3/2, and in the - cases of real-only, imag-only, or summed-only, we need to scale by - 1/2. In both cases, we are compensating for the fact that pointer - arithmetic occurs in terms of complex elements rather than real - elements. */ \ - if ( bli_is_3mi_packed( schema ) ) { ss_num = 3; ss_den = 2; } \ - else if ( bli_is_3ms_packed( schema ) ) { ss_num = 1; ss_den = 2; } \ - else if ( bli_is_rih_packed( schema ) ) { ss_num = 1; ss_den = 2; } \ - else { ss_num = 1; ss_den = 1; } \ \ /* Compute the total number of iterations we'll need. */ \ num_iter = iter_dim / panel_dim_max + ( iter_dim % panel_dim_max ? 1 : 0 ); \ @@ -362,15 +297,6 @@ void PASTEMAC(ch,varname)( \ } \ \ p_begin = p_cast; \ -\ -/* -if ( row_stored ) \ -PASTEMAC(ch,fprintm)( stdout, "packm_var2: b", m, n, \ - c_cast, rs_c, cs_c, "%4.1f", "" ); \ -if ( col_stored ) \ -PASTEMAC(ch,fprintm)( stdout, "packm_var2: a", m, n, \ - c_cast, rs_c, cs_c, "%4.1f", "" ); \ -*/ \ \ for ( ic = ic0, ip = ip0, it = 0; it < num_iter; \ ic += ic_inc, ip += ip_inc, it += 1 ) \ @@ -444,7 +370,7 @@ PASTEMAC(ch,fprintm)( stdout, "packm_var2: a", m, n, \ kappa_cast, \ c_use, rs_c, cs_c, \ p_use, rs_p, cs_p, \ - is_p ); \ + is_p ); \ } \ \ /* NOTE: This value is usually LESS than ps_p because triangular @@ -454,17 +380,12 @@ PASTEMAC(ch,fprintm)( stdout, "packm_var2: a", m, n, \ \ /* We nudge the panel increment up by one if it is odd. */ \ p_inc += ( bli_is_odd( p_inc ) ? 1 : 0 ); \ -\ - p_inc = ( p_inc * ss_num ) / ss_den; \ } \ else if ( bli_is_herm_or_symm( strucc ) ) \ { \ /* This case executes if the panel belongs to a Hermitian or symmetric matrix, which includes stored, unstored, and diagonal-intersecting panels. */ \ -\ - c_use = c_begin; \ - p_use = p_begin; \ \ panel_len_i = panel_len_full; \ panel_len_max_i = panel_len_max; \ @@ -483,12 +404,13 @@ PASTEMAC(ch,fprintm)( stdout, "packm_var2: a", m, n, \ *m_panel_max, \ *n_panel_max, \ kappa_cast, \ - c_use, rs_c, cs_c, \ - p_use, rs_p, cs_p, \ - is_p ); \ + c_begin, rs_c, cs_c, \ + p_begin, rs_p, cs_p, \ + is_p ); \ } \ \ /* NOTE: This value is equivalent to ps_p. */ \ + /*p_inc = ldp * panel_len_max_i;*/ \ p_inc = ps_p; \ } \ else \ @@ -496,9 +418,6 @@ PASTEMAC(ch,fprintm)( stdout, "packm_var2: a", m, n, \ /* This case executes if the panel is general, or, if the panel is part of a triangular matrix and is neither unstored (ie: zero) nor diagonal-intersecting. */ \ -\ - c_use = c_begin; \ - p_use = p_begin; \ \ panel_len_i = panel_len_full; \ panel_len_max_i = panel_len_max; \ @@ -517,69 +436,28 @@ PASTEMAC(ch,fprintm)( stdout, "packm_var2: a", m, n, \ *m_panel_max, \ *n_panel_max, \ kappa_cast, \ - c_use, rs_c, cs_c, \ - p_use, rs_p, cs_p, \ - is_p ); \ + c_begin, rs_c, cs_c, \ + p_begin, rs_p, cs_p, \ + is_p ); \ } \ -\ - /* NOTE: This value is equivalent to ps_p. */ \ - p_inc = ps_p; \ - } \ -\ -/* - if ( bli_is_ro_packed( schema ) ) { \ - if ( col_stored ) { \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: a_r", *m_panel_use, *n_panel_use, \ - ( ctype_r* )c_use, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: ap_r", *m_panel_max, *n_panel_max, \ - ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ - } \ - if ( row_stored && *n_panel_use == 3 ) { \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: b_r", *m_panel_use, *n_panel_use, \ - ( ctype_r* )c_use, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: bp_r", *m_panel_max, *n_panel_max, \ - ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ - } \ - } \ -*/ \ /* - PASTEMAC(chr,fprintm)( stdout, "packm_var2: bp_rpi", *m_panel_max, *n_panel_max, \ - ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ + if ( row_stored ) \ + PASTEMAC(ch,fprintm)( stdout, "packm_var1: bp copied", panel_len_max_i, panel_dim_max, \ + p_begin, rs_p, cs_p, "%9.2e", "" ); \ + else if ( col_stored ) \ + PASTEMAC(ch,fprintm)( stdout, "packm_var1: ap copied", panel_dim_max, panel_len_max_i, \ + p_begin, rs_p, cs_p, "%9.2e", "" ); \ */ \ \ -\ -/* - if ( row_stored ) { \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: b_r", *m_panel_max, *n_panel_max, \ - ( ctype_r* )c_use, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: b_i", *m_panel_max, *n_panel_max, \ - (( ctype_r* )c_use)+rs_c, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: bp_r", *m_panel_max, *n_panel_max, \ - ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ - inc_t is_b = rs_p * *m_panel_max; \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: bp_i", *m_panel_max, *n_panel_max, \ - ( ctype_r* )p_use + is_b, rs_p, cs_p, "%4.1f", "" ); \ + /* NOTE: This value is equivalent to ps_p. */ \ + /*p_inc = ldp * panel_len_max_i;*/ \ + p_inc = ps_p; \ } \ -*/ \ \ -\ -/* - if ( col_stored ) { \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: a_r", *m_panel_max, *n_panel_max, \ - ( ctype_r* )c_use, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: a_i", *m_panel_max, *n_panel_max, \ - (( ctype_r* )c_use)+rs_c, 2*rs_c, 2*cs_c, "%4.1f", "" ); \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: ap_r", *m_panel_max, *n_panel_max, \ - ( ctype_r* )p_use, rs_p, cs_p, "%4.1f", "" ); \ - PASTEMAC(chr,fprintm)( stdout, "packm_var2: ap_i", *m_panel_max, *n_panel_max, \ - ( ctype_r* )p_use + p_inc, rs_p, cs_p, "%4.1f", "" ); \ - } \ -*/ \ \ p_begin += p_inc; \ -\ } \ } -INSERT_GENTFUNCCO_BASIC( packm_blk_var2, packm_ker_t ) +INSERT_GENTFUNC_BASIC( packm_blk_var1, packm_ker_t ) diff --git a/frame/1m/packm/bli_packm_init.c b/frame/1m/packm/bli_packm_init.c index af8a3c32f5..4c35e02015 100644 --- a/frame/1m/packm/bli_packm_init.c +++ b/frame/1m/packm/bli_packm_init.c @@ -164,9 +164,6 @@ void bli_packm_init( obj_t* a, } -extern blksz_t* gemm_upanel_a_align; -extern blksz_t* gemm_upanel_b_align; - void bli_packm_init_pack( invdiag_t invert_diag, pack_t schema, packord_t pack_ord_if_up, @@ -178,7 +175,6 @@ void bli_packm_init_pack( invdiag_t invert_diag, obj_t* p ) { num_t dt = bli_obj_datatype( *c ); - num_t dt_real = bli_obj_datatype_proj_to_real( *c ); trans_t transc = bli_obj_onlytrans_status( *c ); dim_t m_c = bli_obj_length( *c ); dim_t n_c = bli_obj_width( *c ); @@ -327,7 +323,6 @@ void bli_packm_init_pack( invdiag_t invert_diag, { dim_t m_panel; dim_t ps_p, ps_p_orig; - dim_t upanel_a_align; // The panel dimension (for each datatype) should be equal to the // register blocksize in the m dimension. @@ -361,9 +356,6 @@ void bli_packm_init_pack( invdiag_t invert_diag, // Preserve this early panel stride value for use later, if needed. ps_p_orig = ps_p; - // Query the micro-panel alignment for A. - upanel_a_align = bli_blksz_get_def( dt, gemm_upanel_a_align ); - // Here, we adjust the panel stride, if necessary. Remember: ps_p is // always interpreted as being in units of the datatype of the object // which is not necessarily how the micro-panels will be stored. For @@ -374,26 +366,12 @@ void bli_packm_init_pack( invdiag_t invert_diag, if ( bli_is_3mi_packed( schema ) ) { ps_p = ( ps_p * 3 ) / 2; - - // Align the panel stride according to the micro-panel alignment. - ps_p = bli_align_dim_to_size( ps_p, elem_size_p, upanel_a_align ); } else if ( bli_is_3ms_packed( schema ) || bli_is_ro_packed( schema ) || bli_is_io_packed( schema ) || bli_is_rpi_packed( schema ) ) { - // Acquire the element size of the the real projection of the - // current complex datatype. - siz_t elem_size_p_real = elem_size_p / 2; - - // Acquire the micro-panel alignment for the real projection of - // the current complex datatype. - upanel_a_align = bli_blksz_get_def( dt_real, gemm_upanel_a_align ); - - // Align the panel stride according to the micro-panel alignment. - ps_p = bli_align_dim_to_size( ps_p, elem_size_p_real, upanel_a_align ); - // The division by 2 below assumes that ps_p is an even number. // However, it is possible that, at this point, ps_p is an odd. // If it is indeed odd, we nudge it higher. @@ -408,11 +386,6 @@ void bli_packm_init_pack( invdiag_t invert_diag, // actual stored element, we divide the panel_stride by 2. ps_p = ps_p / 2; } - else - { - // Align the panel stride according to the micro-panel alignment. - ps_p = bli_align_dim_to_size( ps_p, elem_size_p, upanel_a_align ); - } // Set the imaginary stride (in units of fundamental elements) for // 3m and 4m (separated or interleaved). We use ps_p_orig since @@ -442,7 +415,6 @@ void bli_packm_init_pack( invdiag_t invert_diag, { dim_t n_panel; dim_t ps_p, ps_p_orig; - dim_t upanel_b_align; // The panel dimension (for each datatype) should be equal to the // register blocksize in the n dimension. @@ -476,9 +448,6 @@ void bli_packm_init_pack( invdiag_t invert_diag, // Preserve this early panel stride value for use later, if needed. ps_p_orig = ps_p; - // Query the micro-panel alignment for B. - upanel_b_align = bli_blksz_get_def( dt, gemm_upanel_b_align ); - // Here, we adjust the panel stride, if necessary. Remember: ps_p is // always interpreted as being in units of the datatype of the object // which is not necessarily how the micro-panels will be stored. For @@ -489,26 +458,12 @@ void bli_packm_init_pack( invdiag_t invert_diag, if ( bli_is_3mi_packed( schema ) ) { ps_p = ( ps_p * 3 ) / 2; - - // Align the panel stride according to the micro-panel alignment. - ps_p = bli_align_dim_to_size( ps_p, elem_size_p, upanel_b_align ); } else if ( bli_is_3ms_packed( schema ) || bli_is_ro_packed( schema ) || bli_is_io_packed( schema ) || bli_is_rpi_packed( schema ) ) { - // Acquire the element size of the the real projection of the - // current complex datatype. - siz_t elem_size_p_real = elem_size_p / 2; - - // Acquire the micro-panel alignment for the real projection of - // the current complex datatype. - upanel_b_align = bli_blksz_get_def( dt_real, gemm_upanel_b_align ); - - // Align the panel stride according to the micro-panel alignment. - ps_p = bli_align_dim_to_size( ps_p, elem_size_p_real, upanel_b_align ); - // The division by 2 below assumes that ps_p is an even number. // However, it is possible that, at this point, ps_p is an odd. // If it is indeed odd, we nudge it higher. @@ -523,11 +478,6 @@ void bli_packm_init_pack( invdiag_t invert_diag, // actual stored element, we divide the panel_stride by 2. ps_p = ps_p / 2; } - else - { - // Align the panel stride according to the micro-panel alignment. - ps_p = bli_align_dim_to_size( ps_p, elem_size_p, upanel_b_align ); - } // Set the imaginary stride (in units of fundamental elements) for // 3m and 4m (separated or interleaved). We use ps_p_orig since diff --git a/frame/1m/packm/bli_packm_int.c b/frame/1m/packm/bli_packm_int.c index 2a82871090..c90c0eaa57 100644 --- a/frame/1m/packm/bli_packm_int.c +++ b/frame/1m/packm/bli_packm_int.c @@ -44,7 +44,7 @@ static FUNCPTR_T vars[6][3] = { // unblocked optimized unblocked blocked { bli_packm_unb_var1, NULL, bli_packm_blk_var1 }, - { NULL, NULL, bli_packm_blk_var2 }, + { NULL, NULL, NULL, }, { NULL, NULL, NULL, }, { NULL, NULL, NULL, }, { NULL, NULL, NULL, }, diff --git a/frame/1m/packm/bli_packm_struc_cxk_3mis.c b/frame/1m/packm/bli_packm_struc_cxk_3mis.c index 48de808d55..1ad507ec68 100644 --- a/frame/1m/packm/bli_packm_struc_cxk_3mis.c +++ b/frame/1m/packm/bli_packm_struc_cxk_3mis.c @@ -57,7 +57,6 @@ void PASTEMAC(ch,varname)( \ { \ dim_t panel_dim; \ dim_t panel_len; \ - dim_t panel_len_max; \ inc_t incc, ldc; \ inc_t ldp; \ \ @@ -69,7 +68,6 @@ void PASTEMAC(ch,varname)( \ /* Prepare to pack to row-stored column panel. */ \ panel_dim = n_panel; \ panel_len = m_panel; \ - panel_len_max = m_panel_max; \ incc = cs_c; \ ldc = rs_c; \ ldp = rs_p; \ @@ -79,7 +77,6 @@ void PASTEMAC(ch,varname)( \ /* Prepare to pack to column-stored row panel. */ \ panel_dim = m_panel; \ panel_len = n_panel; \ - panel_len_max = n_panel_max; \ incc = rs_c; \ ldc = cs_c; \ ldp = cs_p; \ @@ -122,11 +119,6 @@ void PASTEMAC(ch,varname)( \ } \ else /* ( bli_is_triangular( strucc ) ) */ \ { \ - /* We need to re-compute the imaginary stride as a function of - panel_len_max since triangular packed matrices have panels - of varying lengths. */ \ - is_p = ldp * panel_len_max; \ -\ /* Call a helper function for micro-panels of triangular matrices. */ \ PASTEMAC(ch,packm_tri_cxk_3mis)( strucc, \ diff --git a/frame/1m/packm/bli_packm_struc_cxk_4mi.c b/frame/1m/packm/bli_packm_struc_cxk_4mi.c index aa5babfe73..e84ff23e4b 100644 --- a/frame/1m/packm/bli_packm_struc_cxk_4mi.c +++ b/frame/1m/packm/bli_packm_struc_cxk_4mi.c @@ -57,7 +57,6 @@ void PASTEMAC(ch,varname)( \ { \ dim_t panel_dim; \ dim_t panel_len; \ - dim_t panel_len_max; \ inc_t incc, ldc; \ inc_t ldp; \ \ @@ -69,7 +68,6 @@ void PASTEMAC(ch,varname)( \ /* Prepare to pack to row-stored column panel. */ \ panel_dim = n_panel; \ panel_len = m_panel; \ - panel_len_max = m_panel_max; \ incc = cs_c; \ ldc = rs_c; \ ldp = rs_p; \ @@ -79,7 +77,6 @@ void PASTEMAC(ch,varname)( \ /* Prepare to pack to column-stored row panel. */ \ panel_dim = m_panel; \ panel_len = n_panel; \ - panel_len_max = n_panel_max; \ incc = rs_c; \ ldc = cs_c; \ ldp = cs_p; \ @@ -122,11 +119,6 @@ void PASTEMAC(ch,varname)( \ } \ else /* ( bli_is_triangular( strucc ) ) */ \ { \ - /* We need to re-compute the imaginary stride as a function of - panel_len_max since triangular packed matrices have panels - of varying lengths. */ \ - is_p = ldp * panel_len_max; \ -\ /* Call a helper function for micro-panels of triangular matrices. */ \ PASTEMAC(ch,packm_tri_cxk_4mi)( strucc, \ diff --git a/frame/3/gemm/bli_gemm_blk_var1f.c b/frame/3/gemm/bli_gemm_blk_var1f.c index f469154745..f3dfa284b2 100644 --- a/frame/3/gemm/bli_gemm_blk_var1f.c +++ b/frame/3/gemm/bli_gemm_blk_var1f.c @@ -44,14 +44,13 @@ void bli_gemm_blk_var1f( obj_t* a, obj_t b_pack_s; obj_t a1_pack_s, c1_pack_s; - obj_t a1, c1; + obj_t a1, c1; obj_t* a1_pack = NULL; obj_t* b_pack = NULL; obj_t* c1_pack = NULL; dim_t i; dim_t b_alg; - dim_t m_trans; if( thread_am_ochief( thread ) ) { // Initialize object for packing B. @@ -80,21 +79,19 @@ void bli_gemm_blk_var1f( obj_t* a, cntl_sub_packm_b( cntl ), gemm_thread_sub_opackm( thread ) ); - // Query dimension in partitioning direction. - m_trans = bli_obj_length_after_trans( *a ); - dim_t start, end; - bli_get_range( thread, 0, m_trans, - bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), - &start, &end ); + dim_t my_start, my_end; + bli_get_range_t2b( thread, a, + bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), + &my_start, &my_end ); // Partition along the m dimension. - for ( i = start; i < end; i += b_alg ) + for ( i = my_start; i < my_end; i += b_alg ) { // Determine the current algorithmic blocksize. // NOTE: Use of a (for execution datatype) is intentional! // This causes the right blocksize to be used if c and a are // complex and b is real. - b_alg = bli_determine_blocksize_f( i, end, a, + b_alg = bli_determine_blocksize_f( i, my_end, a, cntl_blocksize( cntl ) ); // Acquire partitions for A1 and C1. @@ -130,7 +127,7 @@ void bli_gemm_blk_var1f( obj_t* a, c1_pack, cntl_sub_gemm( cntl ), gemm_thread_sub_gemm( thread ) ); - + thread_ibarrier( thread ); // Unpack C1 (if C1 was packed). diff --git a/frame/3/gemm/bli_gemm_blk_var2f.c b/frame/3/gemm/bli_gemm_blk_var2f.c index b13f21778b..4b3d9da739 100644 --- a/frame/3/gemm/bli_gemm_blk_var2f.c +++ b/frame/3/gemm/bli_gemm_blk_var2f.c @@ -42,7 +42,7 @@ void bli_gemm_blk_var2f( obj_t* a, { obj_t a_pack_s; obj_t b1_pack_s, c1_pack_s; - + obj_t b1, c1; obj_t* a_pack = NULL; obj_t* b1_pack = NULL; @@ -50,7 +50,6 @@ void bli_gemm_blk_var2f( obj_t* a, dim_t i; dim_t b_alg; - dim_t n_trans; if( thread_am_ochief( thread ) ) { @@ -79,21 +78,19 @@ void bli_gemm_blk_var2f( obj_t* a, cntl_sub_packm_a( cntl ), gemm_thread_sub_opackm( thread ) ); - // Query dimension in partitioning direction. - n_trans = bli_obj_width_after_trans( *b ); - dim_t start, end; - bli_get_range( thread, 0, n_trans, - bli_blksz_get_mult_for_obj( b, cntl_blocksize( cntl ) ), - &start, &end ); + dim_t my_start, my_end; + bli_get_range_l2r( thread, b, + bli_blksz_get_mult_for_obj( b, cntl_blocksize( cntl ) ), + &my_start, &my_end ); // Partition along the n dimension. - for ( i = start; i < end; i += b_alg ) + for ( i = my_start; i < my_end; i += b_alg ) { // Determine the current algorithmic blocksize. // NOTE: Use of b (for execution datatype) is intentional! // This causes the right blocksize to be used if c and a are // complex and b is real. - b_alg = bli_determine_blocksize_f( i, end, b, + b_alg = bli_determine_blocksize_f( i, my_end, b, cntl_blocksize( cntl ) ); // Acquire partitions for B1 and C1. @@ -129,7 +126,7 @@ void bli_gemm_blk_var2f( obj_t* a, c1_pack, cntl_sub_gemm( cntl ), gemm_thread_sub_gemm( thread ) ); - + thread_ibarrier( thread ); // Unpack C1 (if C1 was packed). diff --git a/frame/3/gemm/bli_gemm_cntl.c b/frame/3/gemm/bli_gemm_cntl.c index 12e28f4511..bf3e60b611 100644 --- a/frame/3/gemm/bli_gemm_cntl.c +++ b/frame/3/gemm/bli_gemm_cntl.c @@ -43,9 +43,6 @@ blksz_t* gemm_mr; blksz_t* gemm_nr; blksz_t* gemm_kr; -blksz_t* gemm_upanel_a_align; -blksz_t* gemm_upanel_b_align; - func_t* gemm_ukrs; func_t* gemm_ref_ukrs; @@ -100,21 +97,6 @@ void bli_gemm_cntl_init() BLIS_DEFAULT_KR_Z, BLIS_PACKDIM_KR_Z ); - // Create objects for micro-panel alignment (in bytes). - gemm_upanel_a_align - = - bli_blksz_obj_create( BLIS_UPANEL_A_ALIGN_SIZE_S, 0, - BLIS_UPANEL_A_ALIGN_SIZE_D, 0, - BLIS_UPANEL_A_ALIGN_SIZE_C, 0, - BLIS_UPANEL_A_ALIGN_SIZE_Z, 0 ); - gemm_upanel_b_align - = - bli_blksz_obj_create( BLIS_UPANEL_B_ALIGN_SIZE_S, 0, - BLIS_UPANEL_B_ALIGN_SIZE_D, 0, - BLIS_UPANEL_B_ALIGN_SIZE_C, 0, - BLIS_UPANEL_B_ALIGN_SIZE_Z, 0 ); - - // Attach the register blksz_t objects as blocksize multiples to the cache // blksz_t objects. bli_blksz_obj_attach_mult_to( gemm_mr, gemm_mc ); @@ -249,9 +231,6 @@ void bli_gemm_cntl_finalize() bli_blksz_obj_free( gemm_nr ); bli_blksz_obj_free( gemm_kr ); - bli_blksz_obj_free( gemm_upanel_a_align ); - bli_blksz_obj_free( gemm_upanel_b_align ); - bli_func_obj_free( gemm_ukrs ); bli_func_obj_free( gemm_ref_ukrs ); diff --git a/frame/3/gemm/bli_gemm_front.c b/frame/3/gemm/bli_gemm_front.c index 6aa49b706c..78220aff34 100644 --- a/frame/3/gemm/bli_gemm_front.c +++ b/frame/3/gemm/bli_gemm_front.c @@ -97,12 +97,5 @@ void bli_gemm_front( obj_t* alpha, bli_gemm_thrinfo_free_paths( infos, n_threads ); -#ifdef BLIS_ENABLE_FLOP_COUNT - // Increment the global flop counter. - bli_flop_count_inc( 2.0 * bli_obj_length( *c ) - * bli_obj_width( *c ) - * bli_obj_width_after_trans( a_local ) - * ( bli_obj_is_complex( *c ) ? 4.0 : 1.0 ) ); -#endif } diff --git a/frame/3/gemm/ind/bli_gemm_blk_var4f.c b/frame/3/gemm/ind/bli_gemm_blk_var4f.c index 97065c634d..0102675be1 100644 --- a/frame/3/gemm/ind/bli_gemm_blk_var4f.c +++ b/frame/3/gemm/ind/bli_gemm_blk_var4f.c @@ -52,14 +52,13 @@ void bli_gemm_blk_var4f( obj_t* a, obj_t b_pack_s; obj_t a1_pack_s, c1_pack_s; - obj_t a1, c1; + obj_t a1, c1; obj_t* a1_pack = NULL; obj_t* b_pack = NULL; obj_t* c1_pack = NULL; dim_t i; dim_t b_alg; - dim_t m_trans; if( thread_am_ochief( thread ) ) { // Initialize object for packing B. @@ -88,21 +87,19 @@ void bli_gemm_blk_var4f( obj_t* a, cntl_sub_packm_b( cntl ), gemm_thread_sub_opackm( thread ) ); - // Query dimension in partitioning direction. - m_trans = bli_obj_length_after_trans( *a ); - dim_t start, end; - bli_get_range( thread, 0, m_trans, - bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), - &start, &end ); + dim_t my_start, my_end; + bli_get_range_t2b( thread, a, + bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), + &my_start, &my_end ); // Partition along the m dimension. - for ( i = start; i < end; i += b_alg ) + for ( i = my_start; i < my_end; i += b_alg ) { // Determine the current algorithmic blocksize. // NOTE: Use of a (for execution datatype) is intentional! // This causes the right blocksize to be used if c and a are // complex and b is real. - b_alg = bli_determine_blocksize_f( i, end, a, + b_alg = bli_determine_blocksize_f( i, my_end, a, cntl_blocksize( cntl ) ); // Acquire partitions for A1 and C1. @@ -140,7 +137,7 @@ void bli_gemm_blk_var4f( obj_t* a, c1_pack, cntl_sub_gemm( cntl ), gemm_thread_sub_gemm( thread ) ); - + thread_ibarrier( thread ); // Only apply beta within the first of three subproblems. @@ -167,7 +164,7 @@ void bli_gemm_blk_var4f( obj_t* a, c1_pack, cntl_sub_gemm( cntl ), gemm_thread_sub_gemm( thread ) ); - + thread_ibarrier( thread ); @@ -191,7 +188,7 @@ void bli_gemm_blk_var4f( obj_t* a, c1_pack, cntl_sub_gemm( cntl ), gemm_thread_sub_gemm( thread ) ); - + thread_ibarrier( thread ); diff --git a/frame/3/hemm/bli_hemm_front.c b/frame/3/hemm/bli_hemm_front.c index 4a1c2c36e8..4d941967d6 100644 --- a/frame/3/hemm/bli_hemm_front.c +++ b/frame/3/hemm/bli_hemm_front.c @@ -104,12 +104,5 @@ void bli_hemm_front( side_t side, bli_gemm_thrinfo_free_paths( infos, n_threads ); -#ifdef BLIS_ENABLE_FLOP_COUNT - // Increment the global flop counter. - bli_flop_count_inc( 2.0 * bli_obj_length( *c ) - * bli_obj_width( *c ) - * bli_obj_width_after_trans( a_local ) - * ( bli_obj_is_complex( *c ) ? 4.0 : 1.0 ) ); -#endif } diff --git a/frame/3/her2k/bli_her2k_front.c b/frame/3/her2k/bli_her2k_front.c index 0bc4e91608..25bc59eb7a 100644 --- a/frame/3/her2k/bli_her2k_front.c +++ b/frame/3/her2k/bli_her2k_front.c @@ -156,12 +156,5 @@ void bli_her2k_front( obj_t* alpha, // to zero before returning. bli_setid( &BLIS_ZERO, &c_local ); -#ifdef BLIS_ENABLE_FLOP_COUNT - // Increment the global flop counter. - bli_flop_count_inc( 2.0 * bli_obj_length( *c ) - * bli_obj_width( *c ) - * bli_obj_width_after_trans( a_local ) - * ( bli_obj_is_complex( *c ) ? 4.0 : 1.0 ) ); -#endif } diff --git a/frame/3/herk/bli_herk.h b/frame/3/herk/bli_herk.h index 2387a88243..1129d3ff65 100644 --- a/frame/3/herk/bli_herk.h +++ b/frame/3/herk/bli_herk.h @@ -35,6 +35,7 @@ #include "bli_herk_check.h" #include "bli_herk_front.h" #include "bli_herk_int.h" +#include "bli_herk_prune.h" #include "bli_herk_blk_var1f.h" diff --git a/frame/3/herk/bli_herk_blk_var1f.c b/frame/3/herk/bli_herk_blk_var1f.c index fb87d3102c..a733d29b7e 100644 --- a/frame/3/herk/bli_herk_blk_var1f.c +++ b/frame/3/herk/bli_herk_blk_var1f.c @@ -50,9 +50,11 @@ void bli_herk_blk_var1f( obj_t* a, dim_t i; dim_t b_alg; - dim_t m_trans; - if( thread_am_ochief( thread ) ) { + // Prune any zero region that exists along the partitioning dimension. + bli_herk_prune_unref_mparts_m( a, ah, c ); + + if( thread_am_ochief( thread ) ) { // Initialize object for packing A'. bli_obj_init_pack( &ah_pack_s ); bli_packm_init( ah, &ah_pack_s, @@ -61,9 +63,9 @@ void bli_herk_blk_var1f( obj_t* a, // Scale C by beta (if instructed). // Since scalm doesn't support multithreading yet, must be done by chief thread (ew) bli_scalm_int( &BLIS_ONE, - c, + c, cntl_sub_scalm( cntl ) ); - } + } ah_pack = thread_obroadcast( thread, &ah_pack_s ); // Initialize pack objects that are passed into packm_init() for A and C. @@ -79,18 +81,16 @@ void bli_herk_blk_var1f( obj_t* a, cntl_sub_packm_b( cntl ), herk_thread_sub_opackm( thread ) ); - // Query dimension in partitioning direction. - m_trans = bli_obj_length_after_trans( *c ); - dim_t start, end; - bli_get_range_weighted( thread, 0, m_trans, - bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), - bli_obj_is_upper( *c ), &start, &end ); + dim_t my_start, my_end; + bli_get_range_weighted_t2b( thread, c, + bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), + &my_start, &my_end ); // Partition along the m dimension. - for ( i = start; i < end; i += b_alg ) + for ( i = my_start; i < my_end; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_f( i, end, a, + b_alg = bli_determine_blocksize_f( i, my_end, a, cntl_blocksize( cntl ) ); // Acquire partitions for A1 and C1. diff --git a/frame/3/herk/bli_herk_blk_var2f.c b/frame/3/herk/bli_herk_blk_var2f.c index 2b5f3f35f7..b4a0ea924e 100644 --- a/frame/3/herk/bli_herk_blk_var2f.c +++ b/frame/3/herk/bli_herk_blk_var2f.c @@ -41,23 +41,18 @@ void bli_herk_blk_var2f( obj_t* a, herk_thrinfo_t* thread ) { obj_t a_pack_s; - obj_t ah1_pack_s, c1S_pack_s; + obj_t ah1_pack_s, c1_pack_s; - obj_t ah1, c1, c1S; - obj_t aS_pack; + obj_t ah1, c1; obj_t* a_pack; obj_t* ah1_pack; - obj_t* c1S_pack; + obj_t* c1_pack; dim_t i; dim_t b_alg; - dim_t n_trans; - subpart_t stored_part; - // The upper and lower variants are identical, except for which - // merged subpartition is acquired in the loop body. - if ( bli_obj_is_lower( *c ) ) stored_part = BLIS_SUBPART1B; - else stored_part = BLIS_SUBPART1T; + // Prune any zero region that exists along the partitioning dimension. + bli_herk_prune_unref_mparts_n( a, ah, c ); if( thread_am_ochief( thread ) ) { // Initialize object for packing A @@ -75,30 +70,26 @@ void bli_herk_blk_var2f( obj_t* a, // Initialize pack objects for C and A' that are passed into packm_init(). if( thread_am_ichief( thread ) ) { bli_obj_init_pack( &ah1_pack_s ); - bli_obj_init_pack( &c1S_pack_s ); + bli_obj_init_pack( &c1_pack_s ); } ah1_pack = thread_ibroadcast( thread, &ah1_pack_s ); - c1S_pack = thread_ibroadcast( thread, &c1S_pack_s ); + c1_pack = thread_ibroadcast( thread, &c1_pack_s ); // Pack A (if instructed). bli_packm_int( a, a_pack, cntl_sub_packm_a( cntl ), herk_thread_sub_opackm( thread ) ); - // Query dimension in partitioning direction. - n_trans = bli_obj_width_after_trans( *c ); - dim_t start, end; - - // Needs to be replaced with a weighted range because triangle - bli_get_range_weighted( thread, 0, n_trans, - bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), - bli_obj_is_lower( *c ), &start, &end ); + dim_t my_start, my_end; + bli_get_range_weighted_l2r( thread, c, + bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), + &my_start, &my_end ); // Partition along the n dimension. - for ( i = start; i < end; i += b_alg ) + for ( i = my_start; i < my_end; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_f( i, end, a, + b_alg = bli_determine_blocksize_f( i, my_end, a, cntl_blocksize( cntl ) ); // Acquire partitions for A1' and C1. @@ -107,18 +98,11 @@ void bli_herk_blk_var2f( obj_t* a, bli_acquire_mpart_l2r( BLIS_SUBPART1, i, b_alg, c, &c1 ); - // Partition off the stored region of C1 and the corresponding region - // of A_pack. - bli_acquire_mpart_t2b( stored_part, - i, b_alg, &c1, &c1S ); - bli_acquire_mpart_t2b( stored_part, - i, b_alg, a_pack, &aS_pack ); - // Initialize objects for packing A1' and C1. if( thread_am_ichief( thread ) ) { bli_packm_init( &ah1, ah1_pack, cntl_sub_packm_b( cntl ) ); - bli_packm_init( &c1S, c1S_pack, + bli_packm_init( &c1, c1_pack, cntl_sub_packm_c( cntl ) ); } thread_ibarrier( thread ) ; @@ -129,23 +113,23 @@ void bli_herk_blk_var2f( obj_t* a, herk_thread_sub_ipackm( thread ) ); // Pack C1 (if instructed). - bli_packm_int( &c1S, c1S_pack, + bli_packm_int( &c1, c1_pack, cntl_sub_packm_c( cntl ), herk_thread_sub_ipackm( thread ) ) ; // Perform herk subproblem. bli_herk_int( &BLIS_ONE, - &aS_pack, + a_pack, ah1_pack, &BLIS_ONE, - c1S_pack, + c1_pack, cntl_sub_gemm( cntl ), herk_thread_sub_herk( thread ) ); thread_ibarrier( thread ); // Unpack C1 (if C1 was packed). - bli_unpackm_int( c1S_pack, &c1S, + bli_unpackm_int( c1_pack, &c1, cntl_sub_unpackm_c( cntl ), herk_thread_sub_ipackm( thread ) ); } @@ -157,7 +141,7 @@ void bli_herk_blk_var2f( obj_t* a, bli_packm_release( a_pack, cntl_sub_packm_a( cntl ) ); if( thread_am_ichief( thread ) ) { bli_packm_release( ah1_pack, cntl_sub_packm_b( cntl ) ); - bli_packm_release( c1S_pack, cntl_sub_packm_c( cntl ) ); + bli_packm_release( c1_pack, cntl_sub_packm_c( cntl ) ); } } diff --git a/frame/3/herk/bli_herk_blk_var3f.c b/frame/3/herk/bli_herk_blk_var3f.c index df5ec73be1..fd20c14cda 100644 --- a/frame/3/herk/bli_herk_blk_var3f.c +++ b/frame/3/herk/bli_herk_blk_var3f.c @@ -52,6 +52,9 @@ void bli_herk_blk_var3f( obj_t* a, dim_t b_alg; dim_t k_trans; + // Prune any zero region that exists along the partitioning dimension. + bli_herk_prune_unref_mparts_k( a, ah, c ); + if( thread_am_ochief( thread ) ) { // Initialize object for packing C. bli_obj_init_pack( &c_pack_s ); diff --git a/frame/3/herk/bli_herk_front.c b/frame/3/herk/bli_herk_front.c index 3cd244ac60..1129f0690e 100644 --- a/frame/3/herk/bli_herk_front.c +++ b/frame/3/herk/bli_herk_front.c @@ -110,12 +110,5 @@ void bli_herk_front( obj_t* alpha, // to zero before returning. bli_setid( &BLIS_ZERO, &c_local ); -#ifdef BLIS_ENABLE_FLOP_COUNT - // Increment the global flop counter. - bli_flop_count_inc( 1.0 * bli_obj_length( *c ) - * bli_obj_width( *c ) - * bli_obj_width_after_trans( a_local ) - * ( bli_obj_is_complex( *c ) ? 4.0 : 1.0 ) ); -#endif } diff --git a/frame/1m/packm/bli_packm_blk_var2.h b/frame/3/herk/bli_herk_prune.c similarity index 57% rename from frame/1m/packm/bli_packm_blk_var2.h rename to frame/3/herk/bli_herk_prune.c index 84e7b1cb5e..a02c50a34b 100644 --- a/frame/1m/packm/bli_packm_blk_var2.h +++ b/frame/3/herk/bli_herk_prune.c @@ -32,36 +32,33 @@ */ -void bli_packm_blk_var2( obj_t* c, - obj_t* p, - packm_thrinfo_t* t ); +#include "blis.h" +void bli_herk_prune_unref_mparts_m( obj_t* a, + obj_t* ah, + obj_t* c ) +{ + // Prune any unreferenced part from the subpartition of C (that would + // be encountered from partitioning in the m dimension) and adjust the + // subpartition of A accordingly. + bli_prune_unref_mparts( c, BLIS_M, a, BLIS_M ); +} -#undef GENTPROTCO -#define GENTPROTCO( ctype, ctype_r, ch, chr, varname ) \ -\ -void PASTEMAC(ch,varname)( \ - struc_t strucc, \ - doff_t diagoffc, \ - diag_t diagc, \ - uplo_t uploc, \ - trans_t transc, \ - pack_t schema, \ - bool_t invdiag, \ - bool_t revifup, \ - bool_t reviflo, \ - dim_t m, \ - dim_t n, \ - dim_t m_max, \ - dim_t n_max, \ - void* kappa, \ - void* c, inc_t rs_c, inc_t cs_c, \ - void* p, inc_t rs_p, inc_t cs_p, \ - inc_t is_p, \ - dim_t pd_p, inc_t ps_p, \ - void* packm_ker, \ - packm_thrinfo_t* t \ - ); +void bli_herk_prune_unref_mparts_n( obj_t* a, + obj_t* ah, + obj_t* c ) +{ + // Prune any unreferenced part from the subpartition of C (that would + // be encountered from partitioning in the n dimension) and adjust the + // subpartition of Ah accordingly. + bli_prune_unref_mparts( c, BLIS_N, ah, BLIS_N ); +} -INSERT_GENTPROTCO_BASIC( packm_blk_var2 ) +void bli_herk_prune_unref_mparts_k( obj_t* a, + obj_t* ah, + obj_t* c ) +{ + // As long as A and Ah are general in structure, no pruning should be + // for the k dimension. +} diff --git a/frame/base/bli_flops.c b/frame/3/herk/bli_herk_prune.h similarity index 86% rename from frame/base/bli_flops.c rename to frame/3/herk/bli_herk_prune.h index 19f37ec3ad..4754d15846 100644 --- a/frame/base/bli_flops.c +++ b/frame/3/herk/bli_herk_prune.h @@ -32,22 +32,7 @@ */ -#include "blis.h" - -static double bli_flops_executed = 0.0; - -void bli_flop_count_inc( double new_flops ) -{ - bli_flops_executed += new_flops; -} - -double bli_flop_count( void ) -{ - return bli_flops_executed; -} - -void bli_flop_count_reset( void ) -{ - bli_flops_executed = 0.0; -} +void bli_herk_prune_unref_mparts_m( obj_t* a, obj_t* ah, obj_t* c ); +void bli_herk_prune_unref_mparts_n( obj_t* a, obj_t* ah, obj_t* c ); +void bli_herk_prune_unref_mparts_k( obj_t* a, obj_t* ah, obj_t* c ); diff --git a/frame/3/symm/bli_symm_front.c b/frame/3/symm/bli_symm_front.c index d223708ec1..5583d7e0f3 100644 --- a/frame/3/symm/bli_symm_front.c +++ b/frame/3/symm/bli_symm_front.c @@ -103,12 +103,5 @@ void bli_symm_front( side_t side, bli_gemm_thrinfo_free_paths( infos, n_threads ); -#ifdef BLIS_ENABLE_FLOP_COUNT - // Increment the global flop counter. - bli_flop_count_inc( 2.0 * bli_obj_length( *c ) - * bli_obj_width( *c ) - * bli_obj_width_after_trans( a_local ) - * ( bli_obj_is_complex( *c ) ? 4.0 : 1.0 ) ); -#endif } diff --git a/frame/3/syr2k/bli_syr2k_front.c b/frame/3/syr2k/bli_syr2k_front.c index e3475fc44d..ca7a2a8fe1 100644 --- a/frame/3/syr2k/bli_syr2k_front.c +++ b/frame/3/syr2k/bli_syr2k_front.c @@ -127,12 +127,5 @@ void bli_syr2k_front( obj_t* alpha, bli_herk_thrinfo_free_paths( infos, n_threads ); #endif -#ifdef BLIS_ENABLE_FLOP_COUNT - // Increment the global flop counter. - bli_flop_count_inc( 2.0 * bli_obj_length( *c ) - * bli_obj_width( *c ) - * bli_obj_width_after_trans( a_local ) - * ( bli_obj_is_complex( *c ) ? 4.0 : 1.0 ) ); -#endif } diff --git a/frame/3/syrk/bli_syrk_front.c b/frame/3/syrk/bli_syrk_front.c index 53ba3485dc..46529cde76 100644 --- a/frame/3/syrk/bli_syrk_front.c +++ b/frame/3/syrk/bli_syrk_front.c @@ -96,12 +96,5 @@ void bli_syrk_front( obj_t* alpha, bli_herk_thrinfo_free_paths( infos, n_threads ); -#ifdef BLIS_ENABLE_FLOP_COUNT - // Increment the global flop counter. - bli_flop_count_inc( 1.0 * bli_obj_length( *c ) - * bli_obj_width( *c ) - * bli_obj_width_after_trans( a_local ) - * ( bli_obj_is_complex( *c ) ? 4.0 : 1.0 ) ); -#endif } diff --git a/frame/3/trmm/bli_trmm.h b/frame/3/trmm/bli_trmm.h index e621dd201e..09f407a9f5 100644 --- a/frame/3/trmm/bli_trmm.h +++ b/frame/3/trmm/bli_trmm.h @@ -36,6 +36,7 @@ #include "bli_trmm_check.h" #include "bli_trmm_front.h" #include "bli_trmm_int.h" +#include "bli_trmm_prune.h" #include "bli_trmm_blk_var1f.h" diff --git a/frame/3/trmm/bli_trmm_blk_var1f.c b/frame/3/trmm/bli_trmm_blk_var1f.c index eda24e3fa2..d0127cbe82 100644 --- a/frame/3/trmm/bli_trmm_blk_var1f.c +++ b/frame/3/trmm/bli_trmm_blk_var1f.c @@ -50,8 +50,9 @@ void bli_trmm_blk_var1f( obj_t* a, dim_t i; dim_t b_alg; - dim_t m_trans; - dim_t offA; + + // Prune any zero region that exists along the partitioning dimension. + bli_trmm_prune_unref_mparts_m( a, b, c ); if( thread_am_ochief( thread ) ) { // Initialize object for packing B. @@ -81,28 +82,28 @@ void bli_trmm_blk_var1f( obj_t* a, trmm_thread_sub_opackm( thread ) ); // Set the default length of and offset to the non-zero part of A. - m_trans = bli_obj_length_after_trans( *a ); - offA = 0; + //m_trans = bli_obj_length_after_trans( *a ); + //offA = 0; // If A is lower triangular, we have to adjust where the non-zero part of // A begins. If A is upper triangular, we have to adjust the length of // the non-zero part. If A is general/dense, then we keep the defaults. - if ( bli_obj_is_lower( *a ) ) - offA = bli_abs( bli_obj_diag_offset_after_trans( *a ) ); - else if ( bli_obj_is_upper( *a ) ) - m_trans = bli_abs( bli_obj_diag_offset_after_trans( *a ) ) + - bli_obj_width_after_trans( *a ); + //if ( bli_obj_is_lower( *a ) ) + // offA = bli_abs( bli_obj_diag_offset_after_trans( *a ) ); + //else if ( bli_obj_is_upper( *a ) ) + // m_trans = bli_abs( bli_obj_diag_offset_after_trans( *a ) ) + + // bli_obj_width_after_trans( *a ); - dim_t start, end; - bli_get_range_weighted( thread, offA, m_trans, - bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), - bli_obj_is_upper( *c ), &start, &end ); + dim_t my_start, my_end; + bli_get_range_weighted_t2b( thread, a, + bli_blksz_get_mult_for_obj( a, cntl_blocksize( cntl ) ), + &my_start, &my_end ); // Partition along the m dimension. - for ( i = start; i < end; i += b_alg ) + for ( i = my_start; i < my_end; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_f( i, end, a, + b_alg = bli_determine_blocksize_f( i, my_end, a, cntl_blocksize( cntl ) ); // Acquire partitions for A1 and C1. diff --git a/frame/3/trmm/bli_trmm_blk_var2b.c b/frame/3/trmm/bli_trmm_blk_var2b.c index 6758e1a326..fe3752ffad 100644 --- a/frame/3/trmm/bli_trmm_blk_var2b.c +++ b/frame/3/trmm/bli_trmm_blk_var2b.c @@ -50,8 +50,9 @@ void bli_trmm_blk_var2b( obj_t* a, dim_t i; dim_t b_alg; - dim_t n_trans; + // Prune any zero region that exists along the partitioning dimension. + bli_trmm_prune_unref_mparts_n( a, b, c ); if( thread_am_ochief( thread ) ) { // Initialize object for packing A @@ -79,18 +80,16 @@ void bli_trmm_blk_var2b( obj_t* a, cntl_sub_packm_a( cntl ), trmm_thread_sub_opackm( thread ) ); - // Query dimension in partitioning direction. - n_trans = bli_obj_width_after_trans( *b ); - dim_t start, end; - bli_get_range_weighted( thread, 0, n_trans, - bli_blksz_get_mult_for_obj( b, cntl_blocksize( cntl ) ), - bli_obj_is_upper( *c ), &start, &end ); + dim_t my_start, my_end; + bli_get_range_weighted_r2l( thread, b, + bli_blksz_get_mult_for_obj( b, cntl_blocksize( cntl ) ), + &my_start, &my_end ); // Partition along the n dimension. - for ( i = start; i < end; i += b_alg ) + for ( i = my_start; i < my_end; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_b( i, end, b, + b_alg = bli_determine_blocksize_b( i, my_end, b, cntl_blocksize( cntl ) ); // Acquire partitions for B1 and C1. diff --git a/frame/3/trmm/bli_trmm_blk_var2f.c b/frame/3/trmm/bli_trmm_blk_var2f.c index 3b6d780dfc..66ecb38400 100644 --- a/frame/3/trmm/bli_trmm_blk_var2f.c +++ b/frame/3/trmm/bli_trmm_blk_var2f.c @@ -50,8 +50,9 @@ void bli_trmm_blk_var2f( obj_t* a, dim_t i; dim_t b_alg; - dim_t n_trans; + // Prune any zero region that exists along the partitioning dimension. + bli_trmm_prune_unref_mparts_n( a, b, c ); if( thread_am_ochief( thread ) ) { // Initialize object for packing A @@ -79,18 +80,16 @@ void bli_trmm_blk_var2f( obj_t* a, cntl_sub_packm_a( cntl ), trmm_thread_sub_opackm( thread ) ); - // Query dimension in partitioning direction. - n_trans = bli_obj_width_after_trans( *b ); - dim_t start, end; - bli_get_range_weighted( thread, 0, n_trans, - bli_blksz_get_mult_for_obj( b, cntl_blocksize( cntl ) ), - bli_obj_is_lower( *c ), &start, &end ); + dim_t my_start, my_end; + bli_get_range_weighted_l2r( thread, b, + bli_blksz_get_mult_for_obj( b, cntl_blocksize( cntl ) ), + &my_start, &my_end ); // Partition along the n dimension. - for ( i = start; i < end; i += b_alg ) + for ( i = my_start; i < my_end; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_f( i, end, b, + b_alg = bli_determine_blocksize_f( i, my_end, b, cntl_blocksize( cntl ) ); // Acquire partitions for B1 and C1. diff --git a/frame/3/trmm/bli_trmm_blk_var3b.c b/frame/3/trmm/bli_trmm_blk_var3b.c index a9428edb62..e81073b93a 100644 --- a/frame/3/trmm/bli_trmm_blk_var3b.c +++ b/frame/3/trmm/bli_trmm_blk_var3b.c @@ -52,6 +52,9 @@ void bli_trmm_blk_var3b( obj_t* a, dim_t b_alg; dim_t k_trans; + // Prune any zero region that exists along the partitioning dimension. + bli_trmm_prune_unref_mparts_k( a, b, c ); + if( thread_am_ochief( thread ) ){ // Initialize object for packing C bli_obj_init_pack( &c_pack_s ); diff --git a/frame/3/trmm/bli_trmm_blk_var3f.c b/frame/3/trmm/bli_trmm_blk_var3f.c index cca0318f0a..2b1dfa2861 100644 --- a/frame/3/trmm/bli_trmm_blk_var3f.c +++ b/frame/3/trmm/bli_trmm_blk_var3f.c @@ -52,6 +52,9 @@ void bli_trmm_blk_var3f( obj_t* a, dim_t b_alg; dim_t k_trans; + // Prune any zero region that exists along the partitioning dimension. + bli_trmm_prune_unref_mparts_k( a, b, c ); + if( thread_am_ochief( thread ) ){ // Initialize object for packing C bli_obj_init_pack( &c_pack_s ); diff --git a/frame/3/trmm/bli_trmm_front.c b/frame/3/trmm/bli_trmm_front.c index 2b69df9abf..70bf338f2a 100644 --- a/frame/3/trmm/bli_trmm_front.c +++ b/frame/3/trmm/bli_trmm_front.c @@ -96,6 +96,9 @@ void bli_trmm_front( side_t side, // contiguous columns, or if C is stored by columns and the micro-kernel // prefers contiguous rows, transpose the entire operation to allow the // micro-kernel to access elements of C in its preferred manner. + // NOTE: We disable the optimization for 1x1 matrices since the concept + // of row- vs. column storage breaks down. + if ( !bli_obj_is_1x1( c_local ) ) if ( ( bli_obj_is_row_stored( c_local ) && bli_func_prefers_contig_cols( bli_obj_datatype( c_local ), @@ -144,12 +147,5 @@ void bli_trmm_front( side_t side, bli_trmm_thrinfo_free_paths( infos, n_threads ); -#ifdef BLIS_ENABLE_FLOP_COUNT - // Increment the global flop counter. - bli_flop_count_inc( 1.0 * bli_obj_length( *c ) - * bli_obj_width( *c ) - * bli_obj_width_after_trans( a_local ) - * ( bli_obj_is_complex( *c ) ? 4.0 : 1.0 ) ); -#endif } diff --git a/frame/3/trmm/bli_trmm_ll_ker_var2.c b/frame/3/trmm/bli_trmm_ll_ker_var2.c index acb89dc9dd..a50b05dc49 100644 --- a/frame/3/trmm/bli_trmm_ll_ker_var2.c +++ b/frame/3/trmm/bli_trmm_ll_ker_var2.c @@ -198,6 +198,7 @@ void PASTEMAC(ch,varname)( \ inc_t ss_a_num; \ inc_t ss_a_den; \ inc_t ps_a_cur; \ + inc_t is_a_cur; \ auxinfo_t aux; \ \ /* @@ -297,6 +298,9 @@ void PASTEMAC(ch,varname)( \ \ istep_a = PACKMR * k; \ istep_b = PACKNR * k_full; \ +\ + if ( bli_is_odd( istep_a ) ) istep_a += 1; \ + if ( bli_is_odd( istep_b ) ) istep_b += 1; \ \ /* Save the pack schemas of A and B to the auxinfo_t object. */ \ bli_auxinfo_set_schema_a( schema_a, aux ); \ @@ -353,9 +357,9 @@ void PASTEMAC(ch,varname)( \ \ /* Compute the panel stride for the current diagonal- intersecting micro-panel. */ \ - ps_a_cur = k_a1011 * PACKMR; \ - ps_a_cur += ( bli_is_odd( ps_a_cur ) ? 1 : 0 ); \ - ps_a_cur = ( ps_a_cur * ss_a_num ) / ss_a_den; \ + is_a_cur = k_a1011 * PACKMR; \ + is_a_cur += ( bli_is_odd( is_a_cur ) ? 1 : 0 ); \ + ps_a_cur = ( is_a_cur * ss_a_num ) / ss_a_den; \ \ if ( trmm_l_ir_my_iter( i, ir_thread ) ) { \ \ @@ -378,7 +382,7 @@ void PASTEMAC(ch,varname)( \ \ /* Save the 4m1/3m1 imaginary stride of A to the auxinfo_t object. */ \ - bli_auxinfo_set_is_a( PACKMR * k_a1011, aux ); \ + bli_auxinfo_set_is_a( is_a_cur, aux ); \ \ /* Handle interior and edge cases separately. */ \ if ( m_cur == MR && n_cur == NR ) \ diff --git a/frame/3/trmm/bli_trmm_lu_ker_var2.c b/frame/3/trmm/bli_trmm_lu_ker_var2.c index 2b23fbbab6..35a2cefe2d 100644 --- a/frame/3/trmm/bli_trmm_lu_ker_var2.c +++ b/frame/3/trmm/bli_trmm_lu_ker_var2.c @@ -198,6 +198,7 @@ void PASTEMAC(ch,varname)( \ inc_t ss_a_num; \ inc_t ss_a_den; \ inc_t ps_a_cur; \ + inc_t is_a_cur; \ auxinfo_t aux; \ \ /* @@ -304,6 +305,9 @@ void PASTEMAC(ch,varname)( \ \ istep_a = PACKMR * k; \ istep_b = PACKNR * k_full; \ +\ + if ( bli_is_odd( istep_a ) ) istep_a += 1; \ + if ( bli_is_odd( istep_b ) ) istep_b += 1; \ \ /* Save the pack schemas of A and B to the auxinfo_t object. */ \ bli_auxinfo_set_schema_a( schema_a, aux ); \ @@ -360,9 +364,9 @@ void PASTEMAC(ch,varname)( \ \ /* Compute the panel stride for the current diagonal- intersecting micro-panel. */ \ - ps_a_cur = k_a1112 * PACKMR; \ - ps_a_cur += ( bli_is_odd( ps_a_cur ) ? 1 : 0 ); \ - ps_a_cur = ( ps_a_cur * ss_a_num ) / ss_a_den; \ + is_a_cur = k_a1112 * PACKMR; \ + is_a_cur += ( bli_is_odd( is_a_cur ) ? 1 : 0 ); \ + ps_a_cur = ( is_a_cur * ss_a_num ) / ss_a_den; \ \ if ( trmm_l_ir_my_iter( i, ir_thread ) ) { \ \ @@ -385,7 +389,7 @@ void PASTEMAC(ch,varname)( \ \ /* Save the 4m1/3m1 imaginary stride of A to the auxinfo_t object. */ \ - bli_auxinfo_set_is_a( PACKMR * k_a1112, aux ); \ + bli_auxinfo_set_is_a( is_a_cur, aux ); \ \ /* Handle interior and edge cases separately. */ \ if ( m_cur == MR && n_cur == NR ) \ diff --git a/frame/3/trmm/bli_trmm_prune.c b/frame/3/trmm/bli_trmm_prune.c new file mode 100644 index 0000000000..04c0f621df --- /dev/null +++ b/frame/3/trmm/bli_trmm_prune.c @@ -0,0 +1,71 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + +void bli_trmm_prune_unref_mparts_m( obj_t* a, + obj_t* b, + obj_t* c ) +{ + // Prune any unreferenced part from the subpartition of A (that would + // be encountered from partitioning in the m dimension) and adjust the + // subpartition of C accordingly. + bli_prune_unref_mparts( a, BLIS_M, c, BLIS_M ); +} + +void bli_trmm_prune_unref_mparts_n( obj_t* a, + obj_t* b, + obj_t* c ) +{ + // Prune any unreferenced part from the subpartition of B (that would + // be encountered from partitioning in the n dimension) and adjust the + // subpartition of C accordingly. + bli_prune_unref_mparts( b, BLIS_N, c, BLIS_N ); +} + +void bli_trmm_prune_unref_mparts_k( obj_t* a, + obj_t* b, + obj_t* c ) +{ + // Prune any unreferenced part from the subpartition of A (that would + // be encountered from partitioning in the k dimension) and adjust the + // subpartition of B accordingly. + bli_prune_unref_mparts( a, BLIS_N, b, BLIS_M ); + + // Prune any unreferenced part from the subpartition of B (that would + // be encountered from partitioning in the k dimension) and adjust the + // subpartition of A accordingly. + bli_prune_unref_mparts( b, BLIS_M, a, BLIS_N ); +} + diff --git a/frame/3/trmm/bli_trmm_prune.h b/frame/3/trmm/bli_trmm_prune.h new file mode 100644 index 0000000000..1d065a58ea --- /dev/null +++ b/frame/3/trmm/bli_trmm_prune.h @@ -0,0 +1,38 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +void bli_trmm_prune_unref_mparts_m( obj_t* a, obj_t* b, obj_t* c ); +void bli_trmm_prune_unref_mparts_n( obj_t* a, obj_t* b, obj_t* c ); +void bli_trmm_prune_unref_mparts_k( obj_t* a, obj_t* b, obj_t* c ); + diff --git a/frame/3/trmm/bli_trmm_rl_ker_var2.c b/frame/3/trmm/bli_trmm_rl_ker_var2.c index 550c9a232e..941f7a7f28 100644 --- a/frame/3/trmm/bli_trmm_rl_ker_var2.c +++ b/frame/3/trmm/bli_trmm_rl_ker_var2.c @@ -198,6 +198,7 @@ void PASTEMAC(ch,varname)( \ inc_t ss_b_num; \ inc_t ss_b_den; \ inc_t ps_b_cur; \ + inc_t is_b_cur; \ auxinfo_t aux; \ \ /* @@ -304,6 +305,9 @@ void PASTEMAC(ch,varname)( \ \ istep_a = PACKMR * k_full; \ istep_b = PACKNR * k; \ +\ + if ( bli_is_odd( istep_a ) ) istep_a += 1; \ + if ( bli_is_odd( istep_b ) ) istep_b += 1; \ \ /* Save the pack schemas of A and B to the auxinfo_t object. */ \ bli_auxinfo_set_schema_a( schema_a, aux ); \ @@ -350,15 +354,15 @@ void PASTEMAC(ch,varname)( \ { \ /* Compute the panel stride for the current diagonal- intersecting micro-panel. */ \ - ps_b_cur = k_b1121 * PACKNR; \ - ps_b_cur += ( bli_is_odd( ps_b_cur ) ? 1 : 0 ); \ - ps_b_cur = ( ps_b_cur * ss_b_num ) / ss_b_den; \ + is_b_cur = k_b1121 * PACKNR; \ + is_b_cur += ( bli_is_odd( is_b_cur ) ? 1 : 0 ); \ + ps_b_cur = ( is_b_cur * ss_b_num ) / ss_b_den; \ \ if ( trmm_r_jr_my_iter( j, jr_thread ) ) { \ \ /* Save the 4m1/3m1 imaginary stride of B to the auxinfo_t object. */ \ - bli_auxinfo_set_is_b( PACKNR * k_b1121, aux ); \ + bli_auxinfo_set_is_b( is_b_cur, aux ); \ \ /* Loop over the m dimension (MR rows at a time). */ \ for ( i = 0; i < m_iter; ++i ) \ diff --git a/frame/3/trmm/bli_trmm_ru_ker_var2.c b/frame/3/trmm/bli_trmm_ru_ker_var2.c index 3e4d8dabc5..6d7127f6fe 100644 --- a/frame/3/trmm/bli_trmm_ru_ker_var2.c +++ b/frame/3/trmm/bli_trmm_ru_ker_var2.c @@ -198,6 +198,7 @@ void PASTEMAC(ch,varname)( \ inc_t ss_b_num; \ inc_t ss_b_den; \ inc_t ps_b_cur; \ + inc_t is_b_cur; \ auxinfo_t aux; \ \ /* @@ -305,6 +306,9 @@ void PASTEMAC(ch,varname)( \ \ istep_a = PACKMR * k_full; \ istep_b = PACKNR * k; \ +\ + if ( bli_is_odd( istep_a ) ) istep_a += 1; \ + if ( bli_is_odd( istep_b ) ) istep_b += 1; \ \ /* Save the pack schemas of A and B to the auxinfo_t object. */ \ bli_auxinfo_set_schema_a( schema_a, aux ); \ @@ -350,15 +354,15 @@ void PASTEMAC(ch,varname)( \ { \ /* Compute the panel stride for the current diagonal- intersecting micro-panel. */ \ - ps_b_cur = k_b0111 * PACKNR; \ - ps_b_cur += ( bli_is_odd( ps_b_cur ) ? 1 : 0 ); \ - ps_b_cur = ( ps_b_cur * ss_b_num ) / ss_b_den; \ + is_b_cur = k_b0111 * PACKNR; \ + is_b_cur += ( bli_is_odd( is_b_cur ) ? 1 : 0 ); \ + ps_b_cur = ( is_b_cur * ss_b_num ) / ss_b_den; \ \ if ( trmm_r_jr_my_iter( j, jr_thread ) ) { \ \ /* Save the 4m1/3m1 imaginary stride of B to the auxinfo_t object. */ \ - bli_auxinfo_set_is_b( PACKNR * k_b0111, aux ); \ + bli_auxinfo_set_is_b( is_b_cur, aux ); \ \ /* Loop over the m dimension (MR rows at a time). */ \ for ( i = 0; i < m_iter; ++i ) \ diff --git a/frame/3/trmm3/bli_trmm3_front.c b/frame/3/trmm3/bli_trmm3_front.c index f48d1481a0..0a90f68306 100644 --- a/frame/3/trmm3/bli_trmm3_front.c +++ b/frame/3/trmm3/bli_trmm3_front.c @@ -129,8 +129,9 @@ void bli_trmm3_front( side_t side, bli_obj_set_as_root( b_local ); bli_obj_set_as_root( c_local ); - - trmm_thrinfo_t** infos = bli_create_trmm_thrinfo_paths( FALSE ); + // Notice that, unlike trmm_r, there is no dependency in the jc loop + // for trmm3_r, so we can pass in FALSE for jc_dependency. + trmm_thrinfo_t** infos = bli_create_trmm_thrinfo_paths( FALSE ); dim_t n_threads = thread_num_threads( infos[0] ); // Invoke the internal back-end. @@ -146,12 +147,5 @@ void bli_trmm3_front( side_t side, bli_trmm_thrinfo_free_paths( infos, n_threads ); -#ifdef BLIS_ENABLE_FLOP_COUNT - // Increment the global flop counter. - bli_flop_count_inc( 1.0 * bli_obj_length( *c ) - * bli_obj_width( *c ) - * bli_obj_width_after_trans( a_local ) - * ( bli_obj_is_complex( *c ) ? 4.0 : 1.0 ) ); -#endif } diff --git a/frame/3/trsm/bli_trsm.h b/frame/3/trsm/bli_trsm.h index bc60547528..6245d7286d 100644 --- a/frame/3/trsm/bli_trsm.h +++ b/frame/3/trsm/bli_trsm.h @@ -37,6 +37,7 @@ #include "bli_trsm_check.h" #include "bli_trsm_front.h" #include "bli_trsm_int.h" +#include "bli_trsm_prune.h" #include "bli_gemmtrsm_ukernel.h" #include "bli_trsm_ukernel.h" diff --git a/frame/3/trsm/bli_trsm_blk_var1b.c b/frame/3/trsm/bli_trsm_blk_var1b.c index ca79804a1d..29aad41df1 100644 --- a/frame/3/trsm/bli_trsm_blk_var1b.c +++ b/frame/3/trsm/bli_trsm_blk_var1b.c @@ -49,8 +49,9 @@ void bli_trsm_blk_var1b( obj_t* a, dim_t i; dim_t b_alg; - dim_t m_trans; - dim_t offA; + + // Prune any zero region that exists along the partitioning dimension. + bli_trsm_prune_unref_mparts_m( a, b, c ); // Initialize object for packing B. if( thread_am_ochief( thread ) ) { @@ -71,28 +72,19 @@ void bli_trsm_blk_var1b( obj_t* a, cntl_sub_packm_b( cntl ), trsm_thread_sub_opackm( thread ) ); - // Set the default length of and offset to the non-zero part of A. - m_trans = bli_obj_length_after_trans( *a ); - offA = 0; - - // If A is upper triangular, we have to adjust where the non-zero part of - // A begins. - if ( bli_obj_is_upper( *a ) ) - offA = m_trans - bli_abs( bli_obj_diag_offset_after_trans( *a ) ) - - bli_obj_width_after_trans( *a ); - - dim_t start, end; + dim_t my_start, my_end; num_t dt = bli_obj_execution_datatype( *a ); - bli_get_range( thread, offA, m_trans, - //bli_lcm( bli_info_get_default_nr( BLIS_TRSM, dt ), bli_info_get_default_mr( BLIS_TRSM, dt ) ), - bli_info_get_default_mc( BLIS_TRSM, dt ), - &start, &end ); + dim_t bf = ( bli_obj_root_is_triangular( *a ) ? + bli_info_get_default_mr( BLIS_TRSM, dt ) : + bli_info_get_default_nr( BLIS_TRSM, dt ) ); + bli_get_range_b2t( thread, a, bf, + &my_start, &my_end ); // Partition along the remaining portion of the m dimension. - for ( i = start; i < end; i += b_alg ) + for ( i = my_start; i < my_end; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_b( i, end, a, + b_alg = bli_determine_blocksize_b( i, my_end, a, cntl_blocksize( cntl ) ); // Acquire partitions for A1 and C1. diff --git a/frame/3/trsm/bli_trsm_blk_var1f.c b/frame/3/trsm/bli_trsm_blk_var1f.c index fe4ecb3104..b4a90f4634 100644 --- a/frame/3/trsm/bli_trsm_blk_var1f.c +++ b/frame/3/trsm/bli_trsm_blk_var1f.c @@ -49,8 +49,9 @@ void bli_trsm_blk_var1f( obj_t* a, dim_t i; dim_t b_alg; - dim_t m_trans; - dim_t offA; + + // Prune any zero region that exists along the partitioning dimension. + bli_trsm_prune_unref_mparts_m( a, b, c ); // Initialize object for packing B. if( thread_am_ochief( thread ) ) { @@ -71,27 +72,19 @@ void bli_trsm_blk_var1f( obj_t* a, cntl_sub_packm_b( cntl ), trsm_thread_sub_opackm( thread ) ); - // Set the default length of and offset to the non-zero part of A. - m_trans = bli_obj_length_after_trans( *a ); - offA = 0; - - // If A is lower triangular, we have to adjust where the non-zero part of - // A begins. - if ( bli_obj_is_lower( *a ) ) - offA = bli_abs( bli_obj_diag_offset_after_trans( *a ) ); - - dim_t start, end; + dim_t my_start, my_end; num_t dt = bli_obj_execution_datatype( *a ); - bli_get_range( thread, offA, m_trans, - //bli_lcm( bli_info_get_default_nr( BLIS_TRSM, dt ), bli_info_get_default_mr( BLIS_TRSM, dt ) ), - bli_info_get_default_mc( BLIS_TRSM, dt ), - &start, &end ); + dim_t bf = ( bli_obj_root_is_triangular( *a ) ? + bli_info_get_default_mr( BLIS_TRSM, dt ) : + bli_info_get_default_nr( BLIS_TRSM, dt ) ); + bli_get_range_t2b( thread, a, bf, + &my_start, &my_end ); // Partition along the remaining portion of the m dimension. - for ( i = start; i < end; i += b_alg ) + for ( i = my_start; i < my_end; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_f( i, end, a, + b_alg = bli_determine_blocksize_f( i, my_end, a, cntl_blocksize( cntl ) ); // Acquire partitions for A1 and C1. diff --git a/frame/3/trsm/bli_trsm_blk_var2b.c b/frame/3/trsm/bli_trsm_blk_var2b.c index 82b2ea4c86..6676528271 100644 --- a/frame/3/trsm/bli_trsm_blk_var2b.c +++ b/frame/3/trsm/bli_trsm_blk_var2b.c @@ -50,7 +50,9 @@ void bli_trsm_blk_var2b( obj_t* a, dim_t i; dim_t b_alg; - dim_t n_trans; + + // Prune any zero region that exists along the partitioning dimension. + bli_trsm_prune_unref_mparts_n( a, b, c ); // Initialize pack objects for A that are passed into packm_init(). if( thread_am_ochief( thread ) ) { @@ -78,24 +80,21 @@ void bli_trsm_blk_var2b( obj_t* a, // Pack A (if instructed). bli_packm_int( a, a_pack, cntl_sub_packm_a( cntl ), - trmm_thread_sub_opackm( thread ) ); + trsm_thread_sub_opackm( thread ) ); - // Query dimension in partitioning direction. - n_trans = bli_obj_width_after_trans( *b ); - dim_t start, end; + dim_t my_start, my_end; num_t dt = bli_obj_execution_datatype( *a ); - bli_get_range( thread, 0, n_trans, - //bli_lcm( bli_info_get_default_nr( BLIS_TRSM, dt ), - // bli_info_get_default_mr( BLIS_TRSM, dt ) ), - bli_lcm( bli_blksz_get_nr( dt, cntl_blocksize( cntl ) ), - bli_blksz_get_mr( dt, cntl_blocksize( cntl ) ) ), - &start, &end ); + dim_t bf = ( bli_obj_root_is_triangular( *b ) ? + bli_info_get_default_mr( BLIS_TRSM, dt ) : + bli_info_get_default_nr( BLIS_TRSM, dt ) ); + bli_get_range_r2l( thread, b, bf, + &my_start, &my_end ); // Partition along the n dimension. - for ( i = start; i < end; i += b_alg ) + for ( i = my_start; i < my_end; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_b( i, end, b, + b_alg = bli_determine_blocksize_b( i, my_end, b, cntl_blocksize( cntl ) ); // Acquire partitions for B1 and C1. diff --git a/frame/3/trsm/bli_trsm_blk_var2f.c b/frame/3/trsm/bli_trsm_blk_var2f.c index 05833d2da8..955d11349d 100644 --- a/frame/3/trsm/bli_trsm_blk_var2f.c +++ b/frame/3/trsm/bli_trsm_blk_var2f.c @@ -50,7 +50,9 @@ void bli_trsm_blk_var2f( obj_t* a, dim_t i; dim_t b_alg; - dim_t n_trans; + + // Prune any zero region that exists along the partitioning dimension. + bli_trsm_prune_unref_mparts_n( a, b, c ); // Initialize pack objects for A that are passed into packm_init(). if( thread_am_ochief( thread ) ) { @@ -78,24 +80,21 @@ void bli_trsm_blk_var2f( obj_t* a, // Pack A (if instructed). bli_packm_int( a, a_pack, cntl_sub_packm_a( cntl ), - trmm_thread_sub_opackm( thread ) ); + trsm_thread_sub_opackm( thread ) ); - // Query dimension in partitioning direction. - n_trans = bli_obj_width_after_trans( *b ); - dim_t start, end; + dim_t my_start, my_end; num_t dt = bli_obj_execution_datatype( *a ); - bli_get_range( thread, 0, n_trans, - //bli_lcm( bli_info_get_default_nr( BLIS_TRSM, dt ), - // bli_info_get_default_mr( BLIS_TRSM, dt ) ), - bli_lcm( bli_blksz_get_nr( dt, cntl_blocksize( cntl ) ), - bli_blksz_get_mr( dt, cntl_blocksize( cntl ) ) ), - &start, &end ); + dim_t bf = ( bli_obj_root_is_triangular( *b ) ? + bli_info_get_default_mr( BLIS_TRSM, dt ) : + bli_info_get_default_nr( BLIS_TRSM, dt ) ); + bli_get_range_l2r( thread, b, bf, + &my_start, &my_end ); // Partition along the n dimension. - for ( i = start; i < end; i += b_alg ) + for ( i = my_start; i < my_end; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_f( i, end, b, + b_alg = bli_determine_blocksize_f( i, my_end, b, cntl_blocksize( cntl ) ); // Acquire partitions for B1 and C1. diff --git a/frame/3/trsm/bli_trsm_blk_var3b.c b/frame/3/trsm/bli_trsm_blk_var3b.c index bb4c2eecef..411178487d 100644 --- a/frame/3/trsm/bli_trsm_blk_var3b.c +++ b/frame/3/trsm/bli_trsm_blk_var3b.c @@ -52,6 +52,9 @@ void bli_trsm_blk_var3b( obj_t* a, dim_t b_alg; dim_t k_trans; + // Prune any zero region that exists along the partitioning dimension. + bli_trsm_prune_unref_mparts_k( a, b, c ); + // Initialize pack objects for C that are passed into packm_init(). if( thread_am_ochief( thread ) ) { bli_obj_init_pack( &c_pack_s ); @@ -130,10 +133,10 @@ void bli_trsm_blk_var3b( obj_t* a, // internal alpha scalars on A/B and C are non-zero, we must ensure // that they are only used in the first iteration. thread_ibarrier( thread ); - if ( i == 0 && thread_am_ichief( thread ) ) { + if ( i == 0 && thread_am_ichief( thread ) ) { bli_obj_scalar_reset( a ); bli_obj_scalar_reset( b ); - bli_obj_scalar_reset( c_pack ); + bli_obj_scalar_reset( c_pack ); } } diff --git a/frame/3/trsm/bli_trsm_blk_var3f.c b/frame/3/trsm/bli_trsm_blk_var3f.c index e6c43387b4..ce84201a2f 100644 --- a/frame/3/trsm/bli_trsm_blk_var3f.c +++ b/frame/3/trsm/bli_trsm_blk_var3f.c @@ -52,6 +52,9 @@ void bli_trsm_blk_var3f( obj_t* a, dim_t b_alg; dim_t k_trans; + // Prune any zero region that exists along the partitioning dimension. + bli_trsm_prune_unref_mparts_k( a, b, c ); + // Initialize pack objects for C that are passed into packm_init(). if( thread_am_ochief( thread ) ) { bli_obj_init_pack( &c_pack_s ); @@ -130,10 +133,10 @@ void bli_trsm_blk_var3f( obj_t* a, // internal alpha scalars on A/B and C are non-zero, we must ensure // that they are only used in the first iteration. thread_ibarrier( thread ); - if ( i == 0 && thread_am_ichief( thread ) ) { + if ( i == 0 && thread_am_ichief( thread ) ) { bli_obj_scalar_reset( a ); bli_obj_scalar_reset( b ); - bli_obj_scalar_reset( c_pack ); + bli_obj_scalar_reset( c_pack ); } } diff --git a/frame/3/trsm/bli_trsm_front.c b/frame/3/trsm/bli_trsm_front.c index b26ead61f0..554dc36c62 100644 --- a/frame/3/trsm/bli_trsm_front.c +++ b/frame/3/trsm/bli_trsm_front.c @@ -130,12 +130,5 @@ void bli_trsm_front( side_t side, bli_trsm_thrinfo_free_paths( infos, n_threads ); -#ifdef BLIS_ENABLE_FLOP_COUNT - // Increment the global flop counter. - bli_flop_count_inc( 1.0 * bli_obj_length( *c ) - * bli_obj_width( *c ) - * bli_obj_width_after_trans( a_local ) - * ( bli_obj_is_complex( *c ) ? 4.0 : 1.0 ) ); -#endif } diff --git a/frame/3/trsm/bli_trsm_ll_ker_var2.c b/frame/3/trsm/bli_trsm_ll_ker_var2.c index 957f6f78aa..31384e1fa4 100644 --- a/frame/3/trsm/bli_trsm_ll_ker_var2.c +++ b/frame/3/trsm/bli_trsm_ll_ker_var2.c @@ -210,6 +210,7 @@ void PASTEMAC(ch,varname)( \ inc_t ss_a_num; \ inc_t ss_a_den; \ inc_t ps_a_cur; \ + inc_t is_a_cur; \ auxinfo_t aux; \ \ /* @@ -325,6 +326,9 @@ void PASTEMAC(ch,varname)( \ \ istep_a = PACKMR * k; \ istep_b = PACKNR * k_full; \ +\ + if ( bli_is_odd( istep_a ) ) istep_a += 1; \ + if ( bli_is_odd( istep_b ) ) istep_b += 1; \ \ /* Save the pack schemas of A and B to the auxinfo_t object. */ \ bli_auxinfo_set_schema_a( schema_a, aux ); \ @@ -382,14 +386,15 @@ void PASTEMAC(ch,varname)( \ \ /* Compute the panel stride for the current diagonal- intersecting micro-panel. */ \ - ps_a_cur = k_a1011 * PACKMR; \ - ps_a_cur += ( bli_is_odd( ps_a_cur ) ? 1 : 0 ); \ - ps_a_cur = ( ps_a_cur * ss_a_num ) / ss_a_den; \ + is_a_cur = k_a1011 * PACKMR; \ + is_a_cur += ( bli_is_odd( is_a_cur ) ? 1 : 0 ); \ + ps_a_cur = ( is_a_cur * ss_a_num ) / ss_a_den; \ \ /* Compute the addresses of the panel A10 and the triangular block A11. */ \ a10 = a1; \ - a11 = a1 + ( k_a10 * PACKMR ) / off_scl; \ + /* a11 = a1 + ( k_a10 * PACKMR ) / off_scl; */ \ + bli_ptr_add( a11, a1, k_a10 * PACKMR, off_scl ); \ \ /* Compute the addresses of the panel B01 and the block B11. */ \ @@ -414,7 +419,7 @@ void PASTEMAC(ch,varname)( \ \ /* Save the 4m1/3m1 imaginary stride of A to the auxinfo_t object. */ \ - bli_auxinfo_set_is_a( PACKMR * k_a1011, aux ); \ + bli_auxinfo_set_is_a( is_a_cur, aux ); \ \ /* Handle interior and edge cases separately. */ \ if ( m_cur == MR && n_cur == NR ) \ @@ -515,6 +520,44 @@ void PASTEMAC(ch,varname)( \ } \ \ /* +if ( bli_is_4mi_packed( schema_a ) ){ \ +PASTEMAC(d,fprintm)( stdout, "trsm4m1_ll_ker_var2: b_r before", k, n, \ + ( double* )b, rs_b, 1, "%4.1f", "" ); \ +PASTEMAC(d,fprintm)( stdout, "trsm4m1_ll_ker_var2: b_i before", k, n, \ + ( double* )b+72, rs_b, 1, "%4.1f", "" ); \ +}else{ \ +PASTEMAC(d,fprintm)( stdout, "trsmnat_ll_ker_var2: b_r before", k, n, \ + ( double* )b, 2*rs_b, 2, "%4.1f", "" ); \ +PASTEMAC(d,fprintm)( stdout, "trsmnat_ll_ker_var2: b_i before", k, n, \ + ( double* )b+1, 2*rs_b, 2, "%4.1f", "" ); \ +} \ +*/ \ +\ +/* +PASTEMAC(d,fprintm)( stdout, "trsm_ll_ker_var2: a11p_r computed", MR, MR, \ + ( double* )a11, 1, PACKMR, "%4.1f", "" ); \ +*/ \ +\ +/* +if ( bli_is_4mi_packed( schema_a ) ){ \ +PASTEMAC(d,fprintm)( stdout, "trsm4m1_ll_ker_var2: b_r after", k, n, \ + ( double* )b, rs_b, 1, "%4.1f", "" ); \ +PASTEMAC(d,fprintm)( stdout, "trsm4m1_ll_ker_var2: b_i after", k, n, \ + ( double* )b+72, rs_b, 1, "%4.1f", "" ); \ +}else{ \ +PASTEMAC(d,fprintm)( stdout, "trsmnat_ll_ker_var2: b_r after", k, n, \ + ( double* )b, 2*rs_b, 2, "%4.1f", "" ); \ +PASTEMAC(d,fprintm)( stdout, "trsmnat_ll_ker_var2: b_i after", k, n, \ + ( double* )b+1, 2*rs_b, 2, "%4.1f", "" ); \ +} \ + +PASTEMAC(d,fprintm)( stdout, "trsm_ll_ker_var2: b_r", m, n, \ + ( double* )c, 1, cs_c, "%4.1f", "" ); \ +PASTEMAC(d,fprintm)( stdout, "trsm_ll_ker_var2: b_i", m, n, \ + ( double* )c + 8*9, 1, cs_c, "%4.1f", "" ); \ +*/ \ +\ +/* PASTEMAC(ch,fprintm)( stdout, "trsm_ll_ker_var2: a1 (diag)", MR, k_a1011, a1, 1, MR, "%5.2f", "" ); \ PASTEMAC(ch,fprintm)( stdout, "trsm_ll_ker_var2: a11 (diag)", MR, MR, a11, 1, MR, "%5.2f", "" ); \ PASTEMAC(ch,fprintm)( stdout, "trsm_ll_ker_var2: b1 (diag)", k_a1011, NR, bp_i, NR, 1, "%5.2f", "" ); \ diff --git a/frame/3/trsm/bli_trsm_lu_ker_var2.c b/frame/3/trsm/bli_trsm_lu_ker_var2.c index 9270edda6a..0e61291340 100644 --- a/frame/3/trsm/bli_trsm_lu_ker_var2.c +++ b/frame/3/trsm/bli_trsm_lu_ker_var2.c @@ -211,6 +211,7 @@ void PASTEMAC(ch,varname)( \ inc_t ss_a_num; \ inc_t ss_a_den; \ inc_t ps_a_cur; \ + inc_t is_a_cur; \ auxinfo_t aux; \ \ /* @@ -333,6 +334,9 @@ void PASTEMAC(ch,varname)( \ \ istep_a = PACKMR * k; \ istep_b = PACKNR * k_full; \ +\ + if ( bli_is_odd( istep_a ) ) istep_a += 1; \ + if ( bli_is_odd( istep_b ) ) istep_b += 1; \ \ /* Save the pack schemas of A and B to the auxinfo_t object. */ \ bli_auxinfo_set_schema_a( schema_a, aux ); \ @@ -392,14 +396,15 @@ void PASTEMAC(ch,varname)( \ \ /* Compute the panel stride for the current diagonal- intersecting micro-panel. */ \ - ps_a_cur = k_a1112 * PACKMR; \ - ps_a_cur += ( bli_is_odd( ps_a_cur ) ? 1 : 0 ); \ - ps_a_cur = ( ps_a_cur * ss_a_num ) / ss_a_den; \ + is_a_cur = k_a1112 * PACKMR; \ + is_a_cur += ( bli_is_odd( is_a_cur ) ? 1 : 0 ); \ + ps_a_cur = ( is_a_cur * ss_a_num ) / ss_a_den; \ \ /* Compute the addresses of the triangular block A11 and the panel A12. */ \ a11 = a1; \ - a12 = a1 + ( k_a11 * PACKMR ) / off_scl; \ + /* a12 = a1 + ( k_a11 * PACKMR ) / off_scl; */ \ + bli_ptr_add( a12, a1, k_a11 * PACKMR, off_scl ); \ \ /* Compute the addresses of the panel B01 and the block B11. */ \ @@ -424,7 +429,7 @@ void PASTEMAC(ch,varname)( \ \ /* Save the 4m1/3m1 imaginary stride of A to the auxinfo_t object. */ \ - bli_auxinfo_set_is_a( PACKMR * k_a1112, aux ); \ + bli_auxinfo_set_is_a( is_a_cur, aux ); \ \ /* Handle interior and edge cases separately. */ \ if ( m_cur == MR && n_cur == NR ) \ diff --git a/frame/3/trsm/bli_trsm_prune.c b/frame/3/trsm/bli_trsm_prune.c new file mode 100644 index 0000000000..768ea118d5 --- /dev/null +++ b/frame/3/trsm/bli_trsm_prune.c @@ -0,0 +1,71 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + +void bli_trsm_prune_unref_mparts_m( obj_t* a, + obj_t* b, + obj_t* c ) +{ + // Prune any unreferenced part from the subpartition of A (that would + // be encountered from partitioning in the m dimension) and adjust the + // subpartition of C accordingly. + bli_prune_unref_mparts( a, BLIS_M, c, BLIS_M ); +} + +void bli_trsm_prune_unref_mparts_n( obj_t* a, + obj_t* b, + obj_t* c ) +{ + // Prune any unreferenced part from the subpartition of B (that would + // be encountered from partitioning in the n dimension) and adjust the + // subpartition of C accordingly. + bli_prune_unref_mparts( b, BLIS_N, c, BLIS_N ); +} + +void bli_trsm_prune_unref_mparts_k( obj_t* a, + obj_t* b, + obj_t* c ) +{ + // Prune any unreferenced part from the subpartition of A (that would + // be encountered from partitioning in the k dimension) and adjust the + // subpartition of B accordingly. + bli_prune_unref_mparts( a, BLIS_N, b, BLIS_M ); + + // Prune any unreferenced part from the subpartition of B (that would + // be encountered from partitioning in the k dimension) and adjust the + // subpartition of A accordingly. + bli_prune_unref_mparts( b, BLIS_M, a, BLIS_N ); +} + diff --git a/frame/3/trsm/bli_trsm_prune.h b/frame/3/trsm/bli_trsm_prune.h new file mode 100644 index 0000000000..724db38aa9 --- /dev/null +++ b/frame/3/trsm/bli_trsm_prune.h @@ -0,0 +1,38 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +void bli_trsm_prune_unref_mparts_m( obj_t* a, obj_t* b, obj_t* c ); +void bli_trsm_prune_unref_mparts_n( obj_t* a, obj_t* b, obj_t* c ); +void bli_trsm_prune_unref_mparts_k( obj_t* a, obj_t* b, obj_t* c ); + diff --git a/frame/3/trsm/bli_trsm_rl_ker_var2.c b/frame/3/trsm/bli_trsm_rl_ker_var2.c index b90fd19d22..7a8e974904 100644 --- a/frame/3/trsm/bli_trsm_rl_ker_var2.c +++ b/frame/3/trsm/bli_trsm_rl_ker_var2.c @@ -211,6 +211,7 @@ void PASTEMAC(ch,varname)( \ inc_t ss_b_num; \ inc_t ss_b_den; \ inc_t ps_b_cur; \ + inc_t is_b_cur; \ auxinfo_t aux; \ \ /* @@ -346,6 +347,9 @@ void PASTEMAC(ch,varname)( \ \ istep_a = PACKMR * k_full; \ istep_b = PACKNR * k; \ +\ + if ( bli_is_odd( istep_a ) ) istep_a += 1; \ + if ( bli_is_odd( istep_b ) ) istep_b += 1; \ \ /* Save the pack schemas of A and B to the auxinfo_t object. NOTE: We swap the values for A and B since the triangular @@ -399,18 +403,19 @@ void PASTEMAC(ch,varname)( \ /* Compute the addresses of the triangular block B11 and the panel B21. */ \ b11 = b1; \ - b21 = b1 + ( k_b11 * PACKNR ) / off_scl; \ + /* b21 = b1 + ( k_b11 * PACKNR ) / off_scl; */ \ + bli_ptr_add( b21, b1, k_b11 * PACKNR, off_scl ); \ \ /* Compute the panel stride for the current micro-panel. */ \ - ps_b_cur = k_b1121 * PACKNR; \ - ps_b_cur += ( bli_is_odd( ps_b_cur ) ? 1 : 0 ); \ - ps_b_cur = ( ps_b_cur * ss_b_num ) / ss_b_den; \ + is_b_cur = k_b1121 * PACKNR; \ + is_b_cur += ( bli_is_odd( is_b_cur ) ? 1 : 0 ); \ + ps_b_cur = ( is_b_cur * ss_b_num ) / ss_b_den; \ \ /* Save the 4m1/3m1 imaginary stride of B to the auxinfo_t object. NOTE: We swap the values for A and B since the triangular "A" matrix is actually contained within B. */ \ - bli_auxinfo_set_is_a( PACKNR * k_b1121, aux ); \ + bli_auxinfo_set_is_a( is_b_cur, aux ); \ \ /* Loop over the m dimension (MR rows at a time). */ \ for ( i = 0; i < m_iter; ++i ) \ diff --git a/frame/3/trsm/bli_trsm_ru_ker_var2.c b/frame/3/trsm/bli_trsm_ru_ker_var2.c index 86187be208..bd66d654ec 100644 --- a/frame/3/trsm/bli_trsm_ru_ker_var2.c +++ b/frame/3/trsm/bli_trsm_ru_ker_var2.c @@ -210,6 +210,7 @@ void PASTEMAC(ch,varname)( \ inc_t ss_b_num; \ inc_t ss_b_den; \ inc_t ps_b_cur; \ + inc_t is_b_cur; \ auxinfo_t aux; \ \ /* @@ -341,6 +342,9 @@ void PASTEMAC(ch,varname)( \ \ istep_a = PACKMR * k_full; \ istep_b = PACKNR * k; \ +\ + if ( bli_is_odd( istep_a ) ) istep_a += 1; \ + if ( bli_is_odd( istep_b ) ) istep_b += 1; \ \ /* Save the pack schemas of A and B to the auxinfo_t object. NOTE: We swap the values for A and B since the triangular @@ -392,18 +396,19 @@ void PASTEMAC(ch,varname)( \ /* Compute the addresses of the panel B10 and the triangular block B11. */ \ b01 = b1; \ - b11 = b1 + ( k_b01 * PACKNR ) / off_scl; \ + /* b11 = b1 + ( k_b01 * PACKNR ) / off_scl; */ \ + bli_ptr_add( b11, b1, k_b01 * PACKNR, off_scl ); \ \ /* Compute the panel stride for the current micro-panel. */ \ - ps_b_cur = k_b0111 * PACKNR; \ - ps_b_cur += ( bli_is_odd( ps_b_cur ) ? 1 : 0 ); \ - ps_b_cur = ( ps_b_cur * ss_b_num ) / ss_b_den; \ + is_b_cur = k_b0111 * PACKNR; \ + is_b_cur += ( bli_is_odd( is_b_cur ) ? 1 : 0 ); \ + ps_b_cur = ( is_b_cur * ss_b_num ) / ss_b_den; \ \ /* Save the 4m1/3m1 imaginary stride of B to the auxinfo_t object. NOTE: We swap the values for A and B since the triangular "A" matrix is actually contained within B. */ \ - bli_auxinfo_set_is_a( PACKNR * k_b0111, aux ); \ + bli_auxinfo_set_is_a( is_b_cur, aux ); \ \ /* Loop over the m dimension (MR rows at a time). */ \ for ( i = 0; i < m_iter; ++i ) \ diff --git a/frame/3/trsm/ukernels/bli_gemmtrsm_l_ukr_ref.c b/frame/3/trsm/ukernels/bli_gemmtrsm_l_ukr_ref.c index cdd6ad4594..37f0aa3cd0 100644 --- a/frame/3/trsm/ukernels/bli_gemmtrsm_l_ukr_ref.c +++ b/frame/3/trsm/ukernels/bli_gemmtrsm_l_ukr_ref.c @@ -69,6 +69,13 @@ void PASTEMAC(ch,varname)( \ b11, \ c11, rs_c, cs_c, \ data ); \ +\ +/* +PASTEMAC(d,fprintm)( stdout, "gemmtrsm_l_ukr: b0111p_r after", k+3, 8, \ + ( double* )b01, 2*PASTEMAC(ch,packnr), 2, "%4.1f", "" ); \ +PASTEMAC(d,fprintm)( stdout, "gemmtrsm_l_ukr: b0111p_i after", k+3, 8, \ + ( double* )b01 + 1, 2*PASTEMAC(ch,packnr), 2, "%4.1f", "" ); \ +*/ \ } INSERT_GENTFUNC_BASIC2( gemmtrsm_l_ukr_ref, GEMM_UKERNEL, TRSM_L_UKERNEL ) diff --git a/frame/base/bli_check.c b/frame/base/bli_check.c index f2cfdf85ab..d333a40650 100644 --- a/frame/base/bli_check.c +++ b/frame/base/bli_check.c @@ -550,8 +550,10 @@ err_t bli_check_matrix_strides( dim_t m, dim_t n, inc_t rs, inc_t cs, inc_t is ) { if ( rs == 1 && cs == 1 ) { - // Only allow rs == cs == 1 for scalars. - if ( m > 1 || n > 1 ) + // If rs == cs == 1, then we must be dealing with an m-by-1, a + // 1-by-n, or a 1-by-1 matrix and thus at least one of the + // dimensions, m or n, must be unit (even if the other is zero). + if ( m != 1 && n != 1 ) return BLIS_INVALID_DIM_STRIDE_COMBINATION; } else if ( rs == 1 ) diff --git a/frame/include/bli_pool_macro_defs.h b/frame/base/bli_const.c similarity index 56% rename from frame/include/bli_pool_macro_defs.h rename to frame/base/bli_const.c index c958595ae6..4c7d9effc4 100644 --- a/frame/include/bli_pool_macro_defs.h +++ b/frame/base/bli_const.c @@ -32,72 +32,51 @@ */ -#ifndef BLIS_POOL_MACRO_DEFS_H -#define BLIS_POOL_MACRO_DEFS_H - - -// Pool entry query - -#define bli_pool_block_ptrs( pool_p ) \ -\ - ( (pool_p)->block_ptrs ) - -#define bli_pool_num_blocks( pool_p ) \ -\ - ( (pool_p)->num_blocks ) - -#define bli_pool_block_size( pool_p ) \ -\ - ( (pool_p)->block_size ) - -#define bli_pool_top_index( pool_p ) \ -\ - ( (pool_p)->top_index ) - -#define bli_pool_is_exhausted( pool_p ) \ -\ - ( bli_pool_top_index( pool_p ) == -1 ) - - -// Pool entry modification - -#define bli_pool_set_block_ptrs( block_ptrs0, pool_p ) \ -{ \ - (pool_p)->block_ptrs = block_ptrs0; \ +#include "blis.h" + +obj_t BLIS_TWO; +obj_t BLIS_ONE; +obj_t BLIS_ONE_HALF; +obj_t BLIS_ZERO; +obj_t BLIS_MINUS_ONE_HALF; +obj_t BLIS_MINUS_ONE; +obj_t BLIS_MINUS_TWO; + +static bool_t bli_const_is_init = FALSE; + +void bli_const_init( void ) +{ + // If the API is already initialized, return early. + if ( bli_const_is_initialized() ) return; + + bli_obj_create_const( 2.0, &BLIS_TWO ); + bli_obj_create_const( 1.0, &BLIS_ONE ); + bli_obj_create_const( 0.5, &BLIS_ONE_HALF ); + bli_obj_create_const( 0.0, &BLIS_ZERO ); + bli_obj_create_const( -0.5, &BLIS_MINUS_ONE_HALF ); + bli_obj_create_const( -1.0, &BLIS_MINUS_ONE ); + bli_obj_create_const( -2.0, &BLIS_MINUS_TWO ); + + // Mark API as initialized. + bli_const_is_init = TRUE; } -#define bli_pool_set_num_blocks( num_blocks0, pool_p ) \ -{ \ - (pool_p)->num_blocks = num_blocks0; \ +void bli_const_finalize( void ) +{ + bli_obj_free( &BLIS_TWO ); + bli_obj_free( &BLIS_ONE ); + bli_obj_free( &BLIS_ONE_HALF ); + bli_obj_free( &BLIS_ZERO ); + bli_obj_free( &BLIS_MINUS_ONE_HALF ); + bli_obj_free( &BLIS_MINUS_ONE ); + bli_obj_free( &BLIS_MINUS_TWO ); + + // Mark API as uninitialized. + bli_const_is_init = FALSE; } -#define bli_pool_set_block_size( block_size0, pool_p ) \ -{ \ - (pool_p)->block_size = block_size0; \ +bool_t bli_const_is_initialized( void ) +{ + return bli_const_is_init; } -#define bli_pool_set_top_index( top_index0, pool_p ) \ -{ \ - (pool_p)->top_index = top_index0; \ -} - -#define bli_pool_dec_top_index( pool_p ) \ -{ \ - ((pool_p)->top_index)--; \ -} - -#define bli_pool_inc_top_index( pool_p ) \ -{ \ - ((pool_p)->top_index)++; \ -} - -#define bli_pool_init( num_blocks, block_size, block_ptrs, pool_p ) \ -{ \ - bli_pool_set_num_blocks( num_blocks, pool_p ); \ - bli_pool_set_block_size( block_size, pool_p ); \ - bli_pool_set_block_ptrs( block_ptrs, pool_p ); \ - bli_pool_set_top_index( num_blocks - 1, pool_p ); \ -} - - -#endif diff --git a/frame/base/bli_flops.h b/frame/base/bli_const.h similarity index 93% rename from frame/base/bli_flops.h rename to frame/base/bli_const.h index c976fa1cb3..73cfafd10e 100644 --- a/frame/base/bli_flops.h +++ b/frame/base/bli_const.h @@ -32,7 +32,7 @@ */ -void bli_flop_count_inc( double new_flops ); -double bli_flop_count( void ); -void bli_flop_count_reset( void ); +void bli_const_init( void ); +void bli_const_finalize( void ); +bool_t bli_const_is_initialized( void ); diff --git a/frame/base/bli_error.c b/frame/base/bli_error.c index 47634609ce..18c84ab659 100644 --- a/frame/base/bli_error.c +++ b/frame/base/bli_error.c @@ -34,59 +34,36 @@ #include "blis.h" -// Current error checking level. -static errlev_t bli_err_chk_level = BLIS_FULL_ERROR_CHECKING; - -// Internal array to hold error strings. -static char bli_error_string[BLIS_MAX_NUM_ERR_MSGS][BLIS_MAX_ERR_MSG_LENGTH]; - +static bool_t bli_error_is_init = FALSE; -errlev_t bli_error_checking_level() -{ - return bli_err_chk_level; -} - -errlev_t bli_error_checking_level_set( errlev_t new_level ) +void bli_error_init( void ) { - err_t e_val; - errlev_t old_level; - - e_val = bli_check_valid_error_level( new_level ); - bli_check_error_code( e_val ); + // If the API is already initialized, return early. + if ( bli_error_is_initialized() ) return; - old_level = bli_err_chk_level; + bli_error_init_msgs(); - bli_err_chk_level = new_level; - - return old_level; + // Mark API as initialized. + bli_error_is_init = TRUE; } -bool_t bli_error_checking_is_enabled() +void bli_error_finalize( void ) { - return bli_error_checking_level() != BLIS_NO_ERROR_CHECKING; + // Mark API as uninitialized. + bli_error_is_init = FALSE; } -char* bli_error_string_for_code( gint_t code ) +bool_t bli_error_is_initialized( void ) { - return bli_error_string[-code]; + return bli_error_is_init; } -void bli_abort( void ) -{ - fprintf( stderr, "libblis: Aborting.\n" ); - //raise( SIGABRT ); - abort(); -} +// ----------------------------------------------------------------------------- -void bli_print_msg( char* str, char* file, guint_t line ) -{ - fprintf( stderr, "\n" ); - fprintf( stderr, "libblis: %s (line %lu):\n", file, ( long unsigned int )line ); - fprintf( stderr, "libblis: %s\n", str ); - fflush( stderr ); -} +// Internal array to hold error strings. +static char bli_error_string[BLIS_MAX_NUM_ERR_MSGS][BLIS_MAX_ERR_MSG_LENGTH]; -void bli_error_msgs_init( void ) +void bli_error_init_msgs( void ) { sprintf( bli_error_string_for_code(BLIS_INVALID_ERROR_CHECKING_LEVEL), "Invalid error checking level." ); @@ -196,3 +173,53 @@ void bli_error_msgs_init( void ) "Expected object to be alias." ); } +void bli_print_msg( char* str, char* file, guint_t line ) +{ + fprintf( stderr, "\n" ); + fprintf( stderr, "libblis: %s (line %lu):\n", file, ( long unsigned int )line ); + fprintf( stderr, "libblis: %s\n", str ); + fflush( stderr ); +} + +void bli_abort( void ) +{ + fprintf( stderr, "libblis: Aborting.\n" ); + //raise( SIGABRT ); + abort(); +} + +// ----------------------------------------------------------------------------- + +// Current error checking level. +static errlev_t bli_err_chk_level = BLIS_FULL_ERROR_CHECKING; + +errlev_t bli_error_checking_level( void ) +{ + return bli_err_chk_level; +} + +errlev_t bli_error_checking_level_set( errlev_t new_level ) +{ + err_t e_val; + errlev_t old_level; + + e_val = bli_check_valid_error_level( new_level ); + bli_check_error_code( e_val ); + + old_level = bli_err_chk_level; + + bli_err_chk_level = new_level; + + return old_level; +} + +bool_t bli_error_checking_is_enabled( void ) +{ + return bli_error_checking_level() != BLIS_NO_ERROR_CHECKING; +} + +char* bli_error_string_for_code( gint_t code ) +{ + return bli_error_string[-code]; +} + diff --git a/frame/base/bli_error.h b/frame/base/bli_error.h index acba60a730..1dcc9b0b2a 100644 --- a/frame/base/bli_error.h +++ b/frame/base/bli_error.h @@ -33,6 +33,14 @@ */ +void bli_error_init( void ); +void bli_error_finalize( void ); +bool_t bli_error_is_initialized( void ); + +void bli_error_init_msgs( void ); +void bli_print_msg( char* str, char* file, guint_t line ); +void bli_abort( void ); + errlev_t bli_error_checking_level( void ); errlev_t bli_error_checking_level_set( errlev_t new_level ); @@ -40,8 +48,4 @@ bool_t bli_error_checking_is_enabled( void ); char* bli_error_string_for_code( gint_t code ); -void bli_abort( void ); -void bli_print_msg( char* str, char* file, guint_t line ); - -void bli_error_msgs_init( void ); diff --git a/frame/base/bli_info.c b/frame/base/bli_info.c index 84b863948b..941a92c3b5 100644 --- a/frame/base/bli_info.c +++ b/frame/base/bli_info.c @@ -57,16 +57,12 @@ char* bli_info_get_int_type_size_str( void ) { return bli_int_type_size_s gint_t bli_info_get_int_type_size( void ) { return BLIS_INT_TYPE_SIZE; } gint_t bli_info_get_num_fp_types( void ) { return BLIS_NUM_FP_TYPES; } gint_t bli_info_get_max_type_size( void ) { return BLIS_MAX_TYPE_SIZE; } -gint_t bli_info_get_max_num_threads( void ) { return BLIS_MAX_NUM_THREADS; } -gint_t bli_info_get_num_mc_x_kc_blocks( void ) { return BLIS_NUM_MC_X_KC_BLOCKS; } -gint_t bli_info_get_num_kc_x_nc_blocks( void ) { return BLIS_NUM_KC_X_NC_BLOCKS; } -gint_t bli_info_get_num_mc_x_nc_blocks( void ) { return BLIS_NUM_MC_X_NC_BLOCKS; } -gint_t bli_info_get_max_preload_byte_offset( void ) { return BLIS_MAX_PRELOAD_BYTE_OFFSET; } gint_t bli_info_get_simd_align_size( void ) { return BLIS_SIMD_ALIGN_SIZE; } +gint_t bli_info_get_page_size( void ) { return BLIS_PAGE_SIZE; } gint_t bli_info_get_stack_buf_align_size( void ) { return BLIS_STACK_BUF_ALIGN_SIZE; } gint_t bli_info_get_heap_addr_align_size( void ) { return BLIS_HEAP_ADDR_ALIGN_SIZE; } gint_t bli_info_get_heap_stride_align_size( void ) { return BLIS_HEAP_STRIDE_ALIGN_SIZE; } -gint_t bli_info_get_contig_addr_align_size( void ) { return BLIS_CONTIG_ADDR_ALIGN_SIZE; } +gint_t bli_info_get_pool_addr_align_size( void ) { return BLIS_POOL_ADDR_ALIGN_SIZE; } gint_t bli_info_get_enable_stay_auto_init( void ) { #ifdef BLIS_ENABLE_STAY_AUTO_INITIALIZED @@ -83,6 +79,14 @@ gint_t bli_info_get_enable_blas2blis( void ) return 0; #endif } +gint_t bli_info_get_enable_cblas( void ) +{ +#ifdef BLIS_ENABLE_CBLAS + return 1; +#else + return 0; +#endif +} gint_t bli_info_get_blas2blis_int_type_size( void ) { return BLIS_BLAS2BLIS_INT_TYPE_SIZE; } @@ -112,41 +116,6 @@ gint_t bli_info_get_default_kr( opid_t oper, num_t dt ) { return bli_bsv_get_ava gint_t bli_info_get_packdim_mr( opid_t oper, num_t dt ) { return bli_bsv_get_avail_blksz_max_dt( BLIS_MR, oper, dt ); } gint_t bli_info_get_packdim_nr( opid_t oper, num_t dt ) { return bli_bsv_get_avail_blksz_max_dt( BLIS_NR, oper, dt ); } -// -- Micro-panel alignment -- - -extern blksz_t* gemm_upanel_a_align; -extern blksz_t* gemm_upanel_b_align; - -// Micro-panel alignment of A - -gint_t bli_info_get_upanel_a_align_size( num_t dt ) -{ - if ( bli_is_float ( dt ) ) return bli_info_get_upanel_a_align_size_s(); - else if ( bli_is_double ( dt ) ) return bli_info_get_upanel_a_align_size_d(); - else if ( bli_is_scomplex( dt ) ) return bli_info_get_upanel_a_align_size_c(); - else if ( bli_is_dcomplex( dt ) ) return bli_info_get_upanel_a_align_size_z(); - else return 0; -} -gint_t bli_info_get_upanel_a_align_size_s( void ) { bli_init(); return bli_blksz_get_def( BLIS_FLOAT, gemm_upanel_a_align ); } -gint_t bli_info_get_upanel_a_align_size_d( void ) { bli_init(); return bli_blksz_get_def( BLIS_DOUBLE, gemm_upanel_a_align ); } -gint_t bli_info_get_upanel_a_align_size_c( void ) { bli_init(); return bli_blksz_get_def( BLIS_SCOMPLEX, gemm_upanel_a_align ); } -gint_t bli_info_get_upanel_a_align_size_z( void ) { bli_init(); return bli_blksz_get_def( BLIS_DCOMPLEX, gemm_upanel_a_align ); } - -// Micro-panel alignment of B - -gint_t bli_info_get_upanel_b_align_size( num_t dt ) -{ - if ( bli_is_float ( dt ) ) return bli_info_get_upanel_b_align_size_s(); - else if ( bli_is_double ( dt ) ) return bli_info_get_upanel_b_align_size_d(); - else if ( bli_is_scomplex( dt ) ) return bli_info_get_upanel_b_align_size_c(); - else if ( bli_is_dcomplex( dt ) ) return bli_info_get_upanel_b_align_size_z(); - else return 0; -} -gint_t bli_info_get_upanel_b_align_size_s( void ) { bli_init(); return bli_blksz_get_def( BLIS_FLOAT, gemm_upanel_b_align ); } -gint_t bli_info_get_upanel_b_align_size_d( void ) { bli_init(); return bli_blksz_get_def( BLIS_DOUBLE, gemm_upanel_b_align ); } -gint_t bli_info_get_upanel_b_align_size_c( void ) { bli_init(); return bli_blksz_get_def( BLIS_SCOMPLEX, gemm_upanel_b_align ); } -gint_t bli_info_get_upanel_b_align_size_z( void ) { bli_init(); return bli_blksz_get_def( BLIS_DCOMPLEX, gemm_upanel_b_align ); } - // -- Level-2 cache blocksizes -- @@ -268,9 +237,9 @@ char* bli_info_get_trsm_u_ukr_impl_string( ind_t method, num_t dt ) // -- bli_mem_pool_macro_defs.h ------------------------------------------------ -gint_t bli_info_get_mk_pool_size( void ) { return BLIS_MK_POOL_SIZE; } -gint_t bli_info_get_kn_pool_size( void ) { return BLIS_KN_POOL_SIZE; } -gint_t bli_info_get_mn_pool_size( void ) { return BLIS_MN_POOL_SIZE; } +gint_t bli_info_get_mk_pool_size( void ) { return bli_mem_pool_size( BLIS_BUFFER_FOR_A_BLOCK ); } +gint_t bli_info_get_kn_pool_size( void ) { return bli_mem_pool_size( BLIS_BUFFER_FOR_B_PANEL ); } +gint_t bli_info_get_mn_pool_size( void ) { return bli_mem_pool_size( BLIS_BUFFER_FOR_C_PANEL ); } diff --git a/frame/base/bli_info.h b/frame/base/bli_info.h index cf8437c41e..7d6d80d0a3 100644 --- a/frame/base/bli_info.h +++ b/frame/base/bli_info.h @@ -44,18 +44,15 @@ char* bli_info_get_int_type_size_str( void ); gint_t bli_info_get_int_type_size( void ); gint_t bli_info_get_num_fp_types( void ); gint_t bli_info_get_max_type_size( void ); -gint_t bli_info_get_max_num_threads( void ); -gint_t bli_info_get_num_mc_x_kc_blocks( void ); -gint_t bli_info_get_num_kc_x_nc_blocks( void ); -gint_t bli_info_get_num_mc_x_nc_blocks( void ); -gint_t bli_info_get_max_preload_byte_offset( void ); gint_t bli_info_get_simd_align_size( void ); +gint_t bli_info_get_page_size( void ); gint_t bli_info_get_stack_buf_align_size( void ); gint_t bli_info_get_heap_addr_align_size( void ); gint_t bli_info_get_heap_stride_align_size( void ); -gint_t bli_info_get_contig_addr_align_size( void ); +gint_t bli_info_get_pool_addr_align_size( void ); gint_t bli_info_get_enable_stay_auto_init( void ); gint_t bli_info_get_enable_blas2blis( void ); +gint_t bli_info_get_enable_cblas( void ); gint_t bli_info_get_blas2blis_int_type_size( void ); @@ -84,22 +81,6 @@ gint_t bli_info_get_default_nr( opid_t oper, num_t dt ); gint_t bli_info_get_packdim_mr( opid_t oper, num_t dt ); gint_t bli_info_get_packdim_nr( opid_t oper, num_t dt ); -// -- Micro-panel alignment for A -- - -gint_t bli_info_get_upanel_a_align_size( num_t dt ); -gint_t bli_info_get_upanel_a_align_size_s( void ); -gint_t bli_info_get_upanel_a_align_size_d( void ); -gint_t bli_info_get_upanel_a_align_size_c( void ); -gint_t bli_info_get_upanel_a_align_size_z( void ); - -// -- Micro-panel alignment for B -- - -gint_t bli_info_get_upanel_b_align_size( num_t dt ); -gint_t bli_info_get_upanel_b_align_size_s( void ); -gint_t bli_info_get_upanel_b_align_size_d( void ); -gint_t bli_info_get_upanel_b_align_size_c( void ); -gint_t bli_info_get_upanel_b_align_size_z( void ); - // -- Level-2 cache blocksizes -- diff --git a/frame/base/bli_init.c b/frame/base/bli_init.c index ae7e8ba280..6e793fa404 100644 --- a/frame/base/bli_init.c +++ b/frame/base/bli_init.c @@ -34,41 +34,26 @@ #include "blis.h" -// -- Global variables -- - -static bool_t bli_initialized = FALSE; - -obj_t BLIS_TWO; -obj_t BLIS_ONE; -obj_t BLIS_ONE_HALF; -obj_t BLIS_ZERO; -obj_t BLIS_MINUS_ONE_HALF; -obj_t BLIS_MINUS_ONE; -obj_t BLIS_MINUS_TWO; - -packm_thrinfo_t BLIS_PACKM_SINGLE_THREADED; -gemm_thrinfo_t BLIS_GEMM_SINGLE_THREADED; -herk_thrinfo_t BLIS_HERK_SINGLE_THREADED; -thread_comm_t BLIS_SINGLE_COMM; - #ifdef BLIS_ENABLE_PTHREADS pthread_mutex_t initialize_mutex = PTHREAD_MUTEX_INITIALIZER; -pthread_mutex_t mem_manager_mutex; #endif +static bool_t bli_is_init = FALSE; + + err_t bli_init( void ) { err_t r_val = BLIS_FAILURE; - // If bli_initialized is TRUE, then we know without a doubt that + // If bli_is_init is TRUE, then we know without a doubt that // BLIS is presently initialized, and thus we can return early. - if ( bli_initialized == TRUE ) return r_val; + if ( bli_is_init == TRUE ) return r_val; - // NOTE: if bli_initialized is FALSE, we cannot be certain that BLIS + // NOTE: if bli_is_init is FALSE, we cannot be certain that BLIS // is ready to be initialized; it may be the case that a thread is // inside the critical section below and is already in the process // of initializing BLIS, but has not yet finished and updated - // bli_initialized accordingly. This boolean asymmetry is important! + // bli_is_init accordingly. This boolean asymmetry is important! // We enclose the bodies of bli_init() and bli_finalize() in a // critical section (both with the same name) so that they can be @@ -78,55 +63,48 @@ err_t bli_init( void ) // reasons), the conditional test below MUST be within the critical // section to prevent a race condition of the type described above. - // BEGIN CRITICAL SECTION -#ifdef BLIS_ENABLE_OPENMP +#ifdef BLIS_ENABLE_OPENMP _Pragma( "omp critical (init)" ) #endif #ifdef BLIS_ENABLE_PTHREADS - pthread_mutex_lock( &mem_manager_mutex ); + pthread_mutex_lock( &initialize_mutex ); #endif - { - // Proceed with initialization only if BLIS is presently uninitialized. - // Since we bli_init() and bli_finalize() use the same named critical - // section, we can be sure that no other thread is either (a) updating - // bli_initialized, or (b) testing bli_initialized within the critical - // section (for the purposes of deciding whether to perform the - // necessary initialization subtasks). - if ( bli_initialized == FALSE ) + // BEGIN CRITICAL SECTION { - bli_init_const(); - - bli_cntl_init(); - - bli_error_msgs_init(); - - bli_mem_init(); - - bli_ind_init(); - - bli_setup_communicator( &BLIS_SINGLE_COMM, 1 ); - bli_setup_packm_single_threaded_info( &BLIS_PACKM_SINGLE_THREADED ); - bli_setup_gemm_single_threaded_info( &BLIS_GEMM_SINGLE_THREADED ); - bli_setup_herk_single_threaded_info( &BLIS_HERK_SINGLE_THREADED ); - // After initialization is complete, mark BLIS as initialized. - bli_initialized = TRUE; - - // Only the thread that actually performs the initialization will - // return "success". - r_val = BLIS_SUCCESS; + // Proceed with initialization only if BLIS is presently uninitialized. + // Since we bli_init() and bli_finalize() use the same named critical + // section, we can be sure that no other thread is either (a) updating + // bli_is_init, or (b) testing bli_is_init within the critical section + // (for the purposes of deciding whether to perform the necessary + // initialization subtasks). + if ( bli_is_init == FALSE ) + { + // Initialize various sub-APIs. + bli_const_init(); + bli_cntl_init(); + bli_error_init(); + bli_mem_init(); + bli_ind_init(); + bli_thread_init(); + + // After initialization is complete, mark BLIS as initialized. + bli_is_init = TRUE; + + //bli_mem_init(); + + // Only the thread that actually performs the initialization will + // return "success". + r_val = BLIS_SUCCESS; + } } - // END CRITICAL SECTION - } -#ifdef BLIS_ENABLE_PTHREADS - pthread_mutex_unlock( &mem_manager_mutex ); -#endif #ifdef BLIS_ENABLE_PTHREADS - pthread_mutex_init( &mem_manager_mutex, NULL ); + pthread_mutex_unlock( &initialize_mutex ); #endif + return r_val; } @@ -134,15 +112,15 @@ err_t bli_finalize( void ) { err_t r_val = BLIS_FAILURE; - // If bli_initialized is FALSE, then we know without a doubt that + // If bli_is_init is FALSE, then we know without a doubt that // BLIS is presently uninitialized, and thus we can return early. - if ( bli_initialized == FALSE ) return r_val; + if ( bli_is_init == FALSE ) return r_val; - // NOTE: if bli_initialized is TRUE, we cannot be certain that BLIS + // NOTE: if bli_is_init is TRUE, we cannot be certain that BLIS // is ready to be finalized; it may be the case that a thread is // inside the critical section below and is already in the process // of finalizing BLIS, but has not yet finished and updated - // bli_initialized accordingly. This boolean asymmetry is important! + // bli_is_init accordingly. This boolean asymmetry is important! // We enclose the bodies of bli_init() and bli_finalize() in a // critical section (both with the same name) so that they can be @@ -152,73 +130,59 @@ err_t bli_finalize( void ) // reasons), the conditional test below MUST be within the critical // section to prevent a race condition of the type described above. - // BEGIN CRITICAL SECTION -#ifdef BLIS_ENABLE_OPENMP +#ifdef BLIS_ENABLE_OPENMP _Pragma( "omp critical (init)" ) #endif #ifdef BLIS_ENABLE_PTHREADS - pthread_mutex_lock( &mem_manager_mutex ); + pthread_mutex_lock( &initialize_mutex ); #endif - { - // Proceed with finalization only if BLIS is presently initialized. - // Since we bli_init() and bli_finalize() use the same named critical - // section, we can be sure that no other thread is either (a) updating - // bli_initialized, or (b) testing bli_initialized within the critical - // section (for the purposes of deciding whether to perform the - // necessary finalization subtasks). - if ( bli_initialized == TRUE ) + // BEGIN CRITICAL SECTION { - bli_finalize_const(); - - bli_cntl_finalize(); - - // Don't need to do anything to finalize error messages. - bli_mem_finalize(); - - // After finalization is complete, mark BLIS as uninitialized. - bli_initialized = FALSE; - - // Only the thread that actually performs the finalization will - // return "success". - r_val = BLIS_SUCCESS; + // Proceed with finalization only if BLIS is presently initialized. + // Since we bli_init() and bli_finalize() use the same named critical + // section, we can be sure that no other thread is either (a) updating + // bli_is_init, or (b) testing bli_is_init within the critical section + // (for the purposes of deciding whether to perform the necessary + // finalization subtasks). + if ( bli_is_init == TRUE ) + { + // Finalize various sub-APIs. + bli_const_finalize(); + bli_cntl_finalize(); + bli_error_finalize(); + bli_mem_finalize(); + bli_ind_finalize(); + bli_thread_finalize(); + + // After finalization is complete, mark BLIS as uninitialized. + bli_is_init = FALSE; + + // Only the thread that actually performs the finalization will + // return "success". + r_val = BLIS_SUCCESS; + } } - // END CRITICAL SECTION - } + #ifdef BLIS_ENABLE_PTHREADS - pthread_mutex_unlock( &mem_manager_mutex ); + pthread_mutex_unlock( &initialize_mutex ); #endif #ifdef BLIS_ENABLE_PTHREADS - pthread_mutex_destroy( &mem_manager_mutex ); + pthread_mutex_destroy( &initialize_mutex ); #endif return r_val; } -void bli_init_const( void ) +bool_t bli_is_initialized( void ) { - bli_obj_create_const( 2.0, &BLIS_TWO ); - bli_obj_create_const( 1.0, &BLIS_ONE ); - bli_obj_create_const( 0.5, &BLIS_ONE_HALF ); - bli_obj_create_const( 0.0, &BLIS_ZERO ); - bli_obj_create_const( -0.5, &BLIS_MINUS_ONE_HALF ); - bli_obj_create_const( -1.0, &BLIS_MINUS_ONE ); - bli_obj_create_const( -2.0, &BLIS_MINUS_TWO ); + return bli_is_init; } -void bli_finalize_const( void ) -{ - bli_obj_free( &BLIS_TWO ); - bli_obj_free( &BLIS_ONE ); - bli_obj_free( &BLIS_ONE_HALF ); - bli_obj_free( &BLIS_ZERO ); - bli_obj_free( &BLIS_MINUS_ONE_HALF ); - bli_obj_free( &BLIS_MINUS_ONE ); - bli_obj_free( &BLIS_MINUS_TWO ); -} +// ----------------------------------------------------------------------------- void bli_init_auto( err_t* init_result ) { diff --git a/frame/base/bli_init.h b/frame/base/bli_init.h index 670d776c8a..d1e9cc0860 100644 --- a/frame/base/bli_init.h +++ b/frame/base/bli_init.h @@ -32,11 +32,9 @@ */ -err_t bli_init( void ); -err_t bli_finalize( void ); +err_t bli_init( void ); +err_t bli_finalize( void ); +bool_t bli_is_initialized( void ); -void bli_init_const( void ); -void bli_finalize_const( void ); - -void bli_init_auto( err_t* init_result ); -void bli_finalize_auto( err_t init_result ); +void bli_init_auto( err_t* init_result ); +void bli_finalize_auto( err_t init_result ); diff --git a/frame/base/bli_mem.c b/frame/base/bli_mem.c index e789f3f1dd..e70fcd372c 100644 --- a/frame/base/bli_mem.c +++ b/frame/base/bli_mem.c @@ -35,140 +35,109 @@ #include "blis.h" #ifdef BLIS_ENABLE_PTHREADS -extern pthread_mutex_t mem_manager_mutex; +pthread_mutex_t mem_manager_mutex = PTHREAD_MUTEX_INITIALIZER; #endif // Declare one memory pool structure for each block size/shape we want to // be able to allocate. - static pool_t pools[3]; -// Physically contiguous memory for each pool. -// -// Generally speaking, the pool sizes are computed in a sub-header of blis.h -// as follows: -// -// BLIS_MK_POOL_SIZE = BLIS_MAXIMUM_MC_? * BLIS_MAXIMUM_KC_? * BLIS_SIZEOF_? -// -// where "?" is the datatype that results in the largest pool size. The -// constants BLIS_KN_POOL_SIZE and BLIS_MN_POOL_SIZE are computed in a -// similar manner. All constants are computed with appropriate "padding" -// to ensure enough space given the alignments required by bli_config.h. -// - -static void* pool_mk_blk_ptrs[ BLIS_NUM_MC_X_KC_BLOCKS ]; -static char pool_mk_mem[ BLIS_MK_POOL_SIZE ]; - -static void* pool_kn_blk_ptrs[ BLIS_NUM_KC_X_NC_BLOCKS ]; -static char pool_kn_mem[ BLIS_KN_POOL_SIZE ]; - -static void* pool_mn_blk_ptrs[ BLIS_NUM_MC_X_NC_BLOCKS ]; -static char pool_mn_mem[ BLIS_MN_POOL_SIZE ]; - - - +// ----------------------------------------------------------------------------- void bli_mem_acquire_m( siz_t req_size, packbuf_t buf_type, mem_t* mem ) { - siz_t block_size; - dim_t pool_index; pool_t* pool; - void** block_ptrs; - void* block; - gint_t i; + pblk_t* pblk; + dim_t pi; + siz_t block_size; + // Make sure the API is initialized. + bli_mem_init(); if ( buf_type == BLIS_BUFFER_FOR_GEN_USE ) { // For general-use buffer requests, such as those used by level-2 - // operations, using bli_malloc() is sufficient, since using - // physically contiguous memory is not as important there. - block = bli_malloc( req_size ); + // operations, using bli_malloc() is sufficient. + void* buf_sys = bli_malloc( req_size ); // Initialize the mem_t object with: // - the address of the memory block, // - the buffer type (a packbuf_t value), and // - the size of the requested region. // NOTE: We do not initialize the pool field since this block did not - // come from a contiguous memory pool. - bli_mem_set_buffer( block, mem ); + // come from a memory pool. + bli_mem_set_buffer( buf_sys, mem ); + bli_mem_set_buf_sys( buf_sys, mem ); bli_mem_set_buf_type( buf_type, mem ); bli_mem_set_size( req_size, mem ); } else { // This branch handles cases where the memory block needs to come - // from one of the contiguous memory pools. + // from an internal memory pool, in which blocks are allocated once + // and then recycled. // Map the requested packed buffer type to a zero-based index, which // we then use to select the corresponding memory pool. - pool_index = bli_packbuf_index( buf_type ); - pool = &pools[ pool_index ]; + pi = bli_packbuf_index( buf_type ); + pool = &pools[ pi ]; // Unconditionally perform error checking on the memory pool. { err_t e_val; // Make sure that the requested matrix size fits inside of a block - // of the corresponding pool. + // of the corresponding pool. If it does not, the pool was somehow + // initialized improperly. e_val = bli_check_requested_block_size_for_pool( req_size, pool ); bli_check_error_code( e_val ); - - // Make sure that the pool contains at least one block to check out - // to the thread. - e_val = bli_check_if_exhausted_pool( pool ); - bli_check_error_code( e_val ); } - // Access the block pointer array from the memory pool data structure. - block_ptrs = bli_pool_block_ptrs( pool ); - + // Extract the address of the pblk_t struct within the mem_t. + pblk = bli_mem_pblk( mem ); - // BEGIN CRITICAL SECTION -#ifdef BLIS_ENABLE_OPENMP - _Pragma( "omp critical (mem)" ) +#ifdef BLIS_ENABLE_OPENMP + _Pragma( "omp critical (mem)" ) #endif #ifdef BLIS_ENABLE_PTHREADS - pthread_mutex_lock( &mem_manager_mutex ); + pthread_mutex_lock( &mem_manager_mutex ); #endif - { - - // Query the index of the contiguous memory block that resides at the - // "top" of the pool. - i = bli_pool_top_index( pool ); - - // Extract the address of the top block from the block pointer array. - block = block_ptrs[i]; - - // Clear the entry from the block pointer array. (This is actually not - // necessary.) - //block_ptrs[i] = NULL; - - // Decrement the top of the memory pool. - bli_pool_dec_top_index( pool ); + // BEGIN CRITICAL SECTION + { + // Checkout a block from the pool. If the pool is exhausted, + // either because it is still empty or because all blocks have + // been checked out already, additional blocks will be allocated + // automatically, as-needed. Note that the addresses are stored + // directly into the mem_t struct since pblk is the address of + // the struct's pblk_t field. + bli_pool_checkout_block( pblk, pool ); + + // Query the size of the blocks in the pool so we can store it in + // the mem_t object. At this point, it is guaranteed to be at + // least as large as req_size. (NOTE: We must perform the query + // within the critical section to ensure that the pool hasn't + // changed, as unlikely as that would be.) + block_size = bli_pool_block_size( pool ); + } // END CRITICAL SECTION - } + #ifdef BLIS_ENABLE_PTHREADS - pthread_mutex_unlock( &mem_manager_mutex ); + pthread_mutex_unlock( &mem_manager_mutex ); #endif - // Query the size of the blocks in the pool so we can store it in the - // mem_t object. - block_size = bli_pool_block_size( pool ); - // Initialize the mem_t object with: - // - the address of the memory block, // - the buffer type (a packbuf_t value), // - the address of the memory pool to which it belongs, and // - the size of the contiguous memory block (NOT the size of the // requested region). - bli_mem_set_buffer( block, mem ); + // The actual addresses (system and aligned) are already stored in + // the mem_t struct's pblk_t field bli_mem_set_buf_type( buf_type, mem ); bli_mem_set_pool( pool, mem ); bli_mem_set_size( block_size, mem ); @@ -180,71 +149,84 @@ void bli_mem_release( mem_t* mem ) { packbuf_t buf_type; pool_t* pool; - void** block_ptrs; - void* block; - gint_t i; + pblk_t* pblk; + siz_t block_size_cur; + siz_t block_size_prev; - // Extract the address of the memory block we are trying to - // release. - block = bli_mem_buffer( mem ); + // Make sure the API is initialized. + bli_mem_init(); // Extract the buffer type so we know what kind of memory was allocated. buf_type = bli_mem_buf_type( mem ); if ( buf_type == BLIS_BUFFER_FOR_GEN_USE ) { + void* buf_sys = bli_mem_buf_sys( mem ); + // For general-use buffers, we allocate with bli_malloc(), and so // here we need to call bli_free(). - bli_free( block ); + bli_free( buf_sys ); } else { - // This branch handles cases where the memory block came from one - // of the contiguous memory pools. - - // Extract the pool from which the block was allocated. + // Extract the address of the pool from which the memory was + // allocated. pool = bli_mem_pool( mem ); - // Extract the block pointer array associated with the pool. - block_ptrs = bli_pool_block_ptrs( pool ); + // Extract the address of the pblk_t struct within the mem_t struct. + pblk = bli_mem_pblk( mem ); + // Query the size of the blocks that were in the pool at the time + // the pblk_t was checked out. (This is used below, in the critical + // section.) + block_size_prev = bli_mem_size( mem ); - // BEGIN CRITICAL SECTION -#ifdef BLIS_ENABLE_OPENMP - _Pragma( "omp critical (mem)" ) +#ifdef BLIS_ENABLE_OPENMP + _Pragma( "omp critical (mem)" ) #endif #ifdef BLIS_ENABLE_PTHREADS - pthread_mutex_lock( &mem_manager_mutex ); + pthread_mutex_lock( &mem_manager_mutex ); #endif - { - - // Increment the top of the memory pool. - bli_pool_inc_top_index( pool ); - // Query the newly incremented top index. - i = bli_pool_top_index( pool ); - - // Place the address of the block back onto the top of the memory pool. - block_ptrs[i] = block; + // BEGIN CRITICAL SECTION + { + // Query the size of the blocks currently in the pool. + block_size_cur = bli_pool_block_size( pool ); + + // If the block size of the pool has changed since the pblk_t + // was checked out, then we need to free the pblk_t rather + // than check it back in. Why? Because the pool's block size + // has (most likely) increased to meet changing needs (example: + // larger cache blocksizes). Thus, the current pblk_t's smaller + // allocated size is of no use anymore. + if ( block_size_cur != block_size_prev ) + { + // Free the pblk_t using the appropriate function in the + // pool API. + bli_pool_free_block( pblk ); + } + else + { + // Check the block back into the pool. + bli_pool_checkin_block( pblk, pool ); + } + } // END CRITICAL SECTION - } + #ifdef BLIS_ENABLE_PTHREADS - pthread_mutex_unlock( &mem_manager_mutex ); + pthread_mutex_unlock( &mem_manager_mutex ); #endif } - - // Clear the mem_t object so that it appears unallocated. We clear: - // - the buffer field, - // - the pool field, and - // - the size field. + // Clear the mem_t object so that it appears unallocated. This clears: + // - the pblk_t struct's fields (ie: the buffer addresses) + // - the pool field + // - the size field // NOTE: We do not clear the buf_type field since there is no // "uninitialized" value for packbuf_t. - bli_mem_set_buffer( NULL, mem ); - bli_mem_set_pool( NULL, mem ); - bli_mem_set_size( 0, mem ); + bli_mem_clear( mem ); } @@ -257,92 +239,399 @@ void bli_mem_acquire_v( siz_t req_size, } - -void bli_mem_init() +siz_t bli_mem_pool_size( packbuf_t buf_type ) { - dim_t index_a; - dim_t index_b; - dim_t index_c; + siz_t r_val; - // Map each of the packbuf_t values to an index starting at zero. - index_a = bli_packbuf_index( BLIS_BUFFER_FOR_A_BLOCK ); - index_b = bli_packbuf_index( BLIS_BUFFER_FOR_B_PANEL ); - index_c = bli_packbuf_index( BLIS_BUFFER_FOR_C_PANEL ); - - // Initialize contiguous memory pool for MC x KC blocks. - bli_mem_init_pool( pool_mk_mem, - BLIS_MK_BLOCK_SIZE, - BLIS_NUM_MC_X_KC_BLOCKS, - pool_mk_blk_ptrs, - &pools[ index_a ] ); - - // Initialize contiguous memory pool for KC x NC blocks. - bli_mem_init_pool( pool_kn_mem, - BLIS_KN_BLOCK_SIZE, - BLIS_NUM_KC_X_NC_BLOCKS, - pool_kn_blk_ptrs, - &pools[ index_b ] ); - - // Initialize contiguous memory pool for MC x NC blocks. - bli_mem_init_pool( pool_mn_mem, - BLIS_MN_BLOCK_SIZE, - BLIS_NUM_MC_X_NC_BLOCKS, - pool_mn_blk_ptrs, - &pools[ index_c ] ); + if ( buf_type == BLIS_BUFFER_FOR_GEN_USE ) + { + // We don't (yet) track the amount of general-purpose + // memory that is currently allocated. + r_val = 0; + } + else + { + dim_t index; + pool_t* pool; + + // Acquire the pointer to the pool corresponding to the buf_type + // provided. + index = bli_packbuf_index( buf_type ); + pool = &(pools[index]); + + // Compute the pool "size" as the product of the block size + // and the number of blocks in the pool. + r_val = bli_pool_block_size( pool ) * + bli_pool_num_blocks( pool ); + } + + return r_val; } +// ----------------------------------------------------------------------------- + +static bool_t bli_mem_is_init = FALSE; -void bli_mem_init_pool( char* pool_mem, - siz_t block_size, - dim_t num_blocks, - void** block_ptrs, - pool_t* pool ) +void bli_mem_init( void ) { - const siz_t align_size = BLIS_CONTIG_ADDR_ALIGN_SIZE; - dim_t i; + // If the initialization flag is TRUE, we know the API is already + // initialized, so we can return early. + if ( bli_mem_is_init == TRUE ) return; + +#ifdef BLIS_ENABLE_OPENMP + _Pragma( "omp critical (mem)" ) +#endif +#ifdef BLIS_ENABLE_PTHREADS + pthread_mutex_lock( &mem_manager_mutex ); +#endif - // If the pool starting address is not already aligned, advance it - // accordingly. - if ( bli_is_unaligned_to( ( uintptr_t )pool_mem, ( uintptr_t )align_size ) ) + // BEGIN CRITICAL SECTION { - // Notice that this works even if the alignment is not a power of two. - pool_mem += ( ( uintptr_t )align_size - - ( ( uintptr_t )pool_mem % align_size ) ); + // Here, we test the initialization flag again. NOTE: THIS IS NOT + // REDUNDANT. This additional test is needed so that other threads + // that may be waiting to acquire the lock do not perform any + // initialization actions once they are finally allowed into this + // critical section. + if ( bli_mem_is_init == FALSE ) + { + // Initialize the memory pools. + bli_mem_init_pools(); + + // After initialization, mark the API as initialized. + bli_mem_is_init = TRUE; + } } + // END CRITICAL SECTION - // Step through the memory pool, beginning with the aligned address - // determined above, assigning pointers to the beginning of each block_size - // bytes to the ith element of the block_ptrs array. - for ( i = 0; i < num_blocks; ++i ) +#ifdef BLIS_ENABLE_PTHREADS + pthread_mutex_unlock( &mem_manager_mutex ); +#endif +} + +void bli_mem_reinit( void ) +{ + +#ifdef BLIS_ENABLE_OPENMP + _Pragma( "omp critical (mem)" ) +#endif +#ifdef BLIS_ENABLE_PTHREADS + pthread_mutex_lock( &mem_manager_mutex ); +#endif + + // BEGIN CRITICAL SECTION { - // Save the address of pool, which is guaranteed to be aligned. - block_ptrs[i] = pool_mem; + // If for some reason the memory pools have not yet been + // initialized (unlikely), we emulate the body of bli_mem_init(). + if ( bli_mem_is_init == FALSE ) + { + // Initialize the memory pools. + bli_mem_init_pools(); - // Advance pool by one block. - pool_mem += block_size; + // After initialization, mark the API as initialized. + bli_mem_is_init = TRUE; + } + else + { + // Reinitialize the memory pools. + bli_mem_reinit_pools(); + } + } + // END CRITICAL SECTION - // Advance pool a bit further if needed in order to get to the - // beginning of an alignment boundary. - if ( bli_is_unaligned_to( ( uintptr_t )pool_mem, ( uintptr_t )align_size ) ) +#ifdef BLIS_ENABLE_PTHREADS + pthread_mutex_unlock( &mem_manager_mutex ); +#endif +} + +void bli_mem_finalize( void ) +{ + // If the initialization flag is FALSE, we know the API is already + // uninitialized, so we can return early. + if ( bli_mem_is_init == FALSE ) return; + +#ifdef BLIS_ENABLE_OPENMP + _Pragma( "omp critical (mem)" ) +#endif +#ifdef BLIS_ENABLE_PTHREADS + pthread_mutex_lock( &mem_manager_mutex ); +#endif + + // BEGIN CRITICAL SECTION + { + // Here, we test the initialization flag again. NOTE: THIS IS NOT + // REDUNDANT. This additional test is needed so that other threads + // that may be waiting to acquire the lock do not perform any + // finalization actions once they are finally allowed into this + // critical section. + if ( bli_mem_is_init == TRUE ) { - pool_mem += ( ( uintptr_t )align_size - - ( ( uintptr_t )pool_mem % align_size ) ); + // Finalize the memory pools. + bli_mem_finalize_pools(); + + // After finalization, mark the API as uninitialized. + bli_mem_is_init = FALSE; } } + // END CRITICAL SECTION - // Now that we have initialized the array of pointers to the individual - // blocks in the pool, we initialize a pool_t data structure so that we - // can easily manage this pool. - bli_pool_init( num_blocks, - block_size, - block_ptrs, - pool ); +#ifdef BLIS_ENABLE_PTHREADS + pthread_mutex_unlock( &mem_manager_mutex ); +#endif } +bool_t bli_mem_is_initialized( void ) +{ + return bli_mem_is_init; +} +// ----------------------------------------------------------------------------- -void bli_mem_finalize() +void bli_mem_init_pools( void ) { - // Nothing to do. + // Map each of the packbuf_t values to an index starting at zero. + dim_t index_a = bli_packbuf_index( BLIS_BUFFER_FOR_A_BLOCK ); + dim_t index_b = bli_packbuf_index( BLIS_BUFFER_FOR_B_PANEL ); + dim_t index_c = bli_packbuf_index( BLIS_BUFFER_FOR_C_PANEL ); + + siz_t align_size = BLIS_POOL_ADDR_ALIGN_SIZE; + + siz_t block_size_a = 0; + siz_t block_size_b = 0; + siz_t block_size_c = 0; + + // Start with empty pools. + dim_t num_blocks_a = 0; + dim_t num_blocks_b = 0; + dim_t num_blocks_c = 0; + + // Determine the block size for each memory pool. + bli_mem_compute_pool_block_sizes( &block_size_a, + &block_size_b, + &block_size_c ); + + // Initialize the memory pools for A, B, and C. + bli_pool_init( num_blocks_a, block_size_a, align_size, &pools[ index_a ] ); + bli_pool_init( num_blocks_b, block_size_b, align_size, &pools[ index_b ] ); + bli_pool_init( num_blocks_c, block_size_c, align_size, &pools[ index_c ] ); +} + +void bli_mem_reinit_pools( void ) +{ + // Map each of the packbuf_t values to an index starting at zero. + dim_t index_a = bli_packbuf_index( BLIS_BUFFER_FOR_A_BLOCK ); + dim_t index_b = bli_packbuf_index( BLIS_BUFFER_FOR_B_PANEL ); + dim_t index_c = bli_packbuf_index( BLIS_BUFFER_FOR_C_PANEL ); + + siz_t align_size = BLIS_POOL_ADDR_ALIGN_SIZE; + + siz_t block_size_a = 0; + siz_t block_size_b = 0; + siz_t block_size_c = 0; + + // Query the number of blocks currently allocated in each pool. + dim_t num_blocks_a = bli_pool_num_blocks( &pools[ index_a ] ); + dim_t num_blocks_b = bli_pool_num_blocks( &pools[ index_b ] ); + dim_t num_blocks_c = bli_pool_num_blocks( &pools[ index_c ] ); + + // Determine the block size for each memory pool. + bli_mem_compute_pool_block_sizes( &block_size_a, + &block_size_b, + &block_size_c ); + + // Reinitialize the memory pools for A, B, and C with the same number + // of blocks as before, except with new block sizes. + bli_pool_reinit( num_blocks_a, block_size_a, align_size, &pools[ index_a ] ); + bli_pool_reinit( num_blocks_b, block_size_b, align_size, &pools[ index_b ] ); + bli_pool_reinit( num_blocks_c, block_size_c, align_size, &pools[ index_c ] ); +} + +void bli_mem_finalize_pools( void ) +{ + // Map each of the packbuf_t values to an index starting at zero. + dim_t index_a = bli_packbuf_index( BLIS_BUFFER_FOR_A_BLOCK ); + dim_t index_b = bli_packbuf_index( BLIS_BUFFER_FOR_B_PANEL ); + dim_t index_c = bli_packbuf_index( BLIS_BUFFER_FOR_C_PANEL ); + + // Finalize the memory pools for A, B, and C. + bli_pool_finalize( &pools[ index_a ] ); + bli_pool_finalize( &pools[ index_b ] ); + bli_pool_finalize( &pools[ index_c ] ); } +// ----------------------------------------------------------------------------- + +void bli_mem_compute_pool_block_sizes( siz_t* bs_a, + siz_t* bs_b, + siz_t* bs_c ) +{ + siz_t bs_cand_a = 0; + siz_t bs_cand_b = 0; + siz_t bs_cand_c = 0; + + ind_t im; + num_t dt; + + // Compute pool block sizes for datatype and each implemented + // method and find the maximum size for each pool. This is done + // so that new pools do not need to be allocated if the user + // switches datatypes or methods later on. + for ( im = 0; im < BLIS_NUM_IND_METHODS; ++im ) + { + for ( dt = BLIS_DT_LO; dt <= BLIS_DT_HI; ++dt ) + { + // Avoid considering induced methods for real datatypes. + if ( bli_is_complex( dt ) || im == BLIS_NAT ) + { + siz_t bs_dt_a, bs_dt_b, bs_dt_c; + + bli_mem_compute_pool_block_sizes_dt( dt, im, + &bs_dt_a, + &bs_dt_b, + &bs_dt_c ); + + bs_cand_a = bli_max( bs_dt_a, bs_cand_a ); + bs_cand_b = bli_max( bs_dt_b, bs_cand_b ); + bs_cand_c = bli_max( bs_dt_c, bs_cand_c ); + } + } + } + + // Save the results. + *bs_a = bs_cand_a; + *bs_b = bs_cand_b; + *bs_c = bs_cand_c; +} + +// ----------------------------------------------------------------------------- + +void bli_mem_compute_pool_block_sizes_dt( num_t dt, + ind_t method, + siz_t* bs_a, + siz_t* bs_b, + siz_t* bs_c ) +{ + siz_t size_dt = bli_datatype_size( dt ); + + blksz_t* mr; + blksz_t* nr; + + blksz_t* mc; + blksz_t* kc; + blksz_t* nc; + + dim_t mr_dt; + dim_t nr_dt; + dim_t max_mnr_dt; + + dim_t mc_max_dt; + dim_t kc_max_dt; + dim_t nc_max_dt; + + dim_t packmr_dt, packnr_dt; + dim_t max_packmnr_dt; + + dim_t scale_num_dt; + dim_t scale_den_dt; + + dim_t pool_mc_dt, left_mc_dt; + dim_t pool_nc_dt, left_nc_dt; + dim_t pool_kc_dt; + + // + // Find the larger of the two register blocksizes. + // + + // Query the mr and nr blksz_t objects for the given method of + // execution. + mr = bli_bsv_get_blksz( BLIS_MR, method ); + nr = bli_bsv_get_blksz( BLIS_NR, method ); + + // Extract the mr and nr values specific to the current datatype. + mr_dt = bli_blksz_get_def( dt, mr ); + nr_dt = bli_blksz_get_def( dt, nr ); + + // Find the maximum of mr and nr. + max_mnr_dt = bli_max( mr_dt, nr_dt ); + + // + // Define local maximum cache blocksizes. + // + + // Query the mc, kc, and nc blksz_t objects for native execution. + mc = bli_bsv_get_blksz( BLIS_MC, method ); + kc = bli_bsv_get_blksz( BLIS_KC, method ); + nc = bli_bsv_get_blksz( BLIS_NC, method ); + + // Extract the maximum mc, kc, and nc values specific to the current + // datatype. + mc_max_dt = bli_blksz_get_max( dt, mc ); + kc_max_dt = bli_blksz_get_max( dt, kc ); + nc_max_dt = bli_blksz_get_max( dt, nc ); + + // Add max(mr,nr) to kc to make room for the nudging of kc at + // runtime to be a multiple of mr or nr for triangular operations + // trmm, trmm3, and trsm. + kc_max_dt += max_mnr_dt; + + // + // Compute scaling factors. + // + + // Compute integer scaling factors (numerator and denominator) used + // to account for situations when the packing register blocksizes are + // larger than the regular register blocksizes. + + // In order to compute the scaling factors, we first have to determine + // whether ( packmr / mr ) is greater than ( packnr / nr ). This is + // needed ONLY because the amount of space allocated for a block of A + // and a panel of B needs to be such that MR and NR can be swapped (ie: + // A is packed with NR and B is packed with MR). This transformation is + // needed for right-side trsm when inducing an algorithm that (a) has + // favorable access patterns for column-stored C and (b) allows the + // macro-kernel to reuse the existing left-side fused gemmtrsm micro- + // kernels. We avoid integer division by cross-multiplying: + // + // ( packmr / mr ) >= ( packnr / nr ) + // ( packmr / mr ) * nr >= packnr + // packmr * nr >= packnr * mr + // + // So, if packmr * nr >= packnr * mr, then we will use packmr and mr as + // our scaling factors. Otherwise, we'll use packnr and nr. + + packmr_dt = bli_blksz_get_max( dt, mr ); + packnr_dt = bli_blksz_get_max( dt, nr ); + + if ( packmr_dt * nr_dt >= + packnr_dt * mr_dt ) { scale_num_dt = packmr_dt; + scale_den_dt = mr_dt; } + else { scale_num_dt = packnr_dt; + scale_den_dt = nr_dt; } + + // + // Compute pool block dimensions. + // + + pool_mc_dt = ( mc_max_dt * scale_num_dt ) / scale_den_dt; + left_mc_dt = ( mc_max_dt * scale_num_dt ) % scale_den_dt; + + pool_nc_dt = ( nc_max_dt * scale_num_dt ) / scale_den_dt; + left_nc_dt = ( nc_max_dt * scale_num_dt ) % scale_den_dt; + + pool_kc_dt = ( kc_max_dt ); + + if ( left_mc_dt > 0 ) pool_mc_dt += 1; + if ( left_nc_dt > 0 ) pool_nc_dt += 1; + + // + // Compute pool block sizes + // + + // We add an extra micro-panel of space to the block sizes for A and B + // just to be sure any pre-loading performed by the micro-kernel does + // not cause a segmentation fault. + max_packmnr_dt = bli_max( packmr_dt, packnr_dt ); + + *bs_a = ( pool_mc_dt + max_packmnr_dt ) * pool_kc_dt * size_dt; + *bs_b = ( pool_nc_dt + max_packmnr_dt ) * pool_kc_dt * size_dt; + *bs_c = ( pool_mc_dt ) * pool_nc_dt * size_dt; +} diff --git a/frame/base/bli_mem.c.prev b/frame/base/bli_mem.c.prev new file mode 100644 index 0000000000..7a16e87327 --- /dev/null +++ b/frame/base/bli_mem.c.prev @@ -0,0 +1,366 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + +#ifdef BLIS_ENABLE_PTHREADS +extern pthread_mutex_t mem_manager_mutex; +#endif + +// Declare one memory pool structure for each block size/shape we want to +// be able to allocate. + +static pool_t pools[3]; + + +// Physically contiguous memory for each pool. +// +// Generally speaking, the pool sizes are computed in a sub-header of blis.h +// as follows: +// +// BLIS_MK_POOL_SIZE = BLIS_MAXIMUM_MC_? * BLIS_MAXIMUM_KC_? * BLIS_SIZEOF_? +// +// where "?" is the datatype that results in the largest pool size. The +// constants BLIS_KN_POOL_SIZE and BLIS_MN_POOL_SIZE are computed in a +// similar manner. All constants are computed with appropriate "padding" +// to ensure enough space given the alignments required by bli_config.h. +// + +static void* pool_mk_blk_ptrs[ BLIS_NUM_MC_X_KC_BLOCKS ]; +static void* pool_kn_blk_ptrs[ BLIS_NUM_KC_X_NC_BLOCKS ]; +static void* pool_mn_blk_ptrs[ BLIS_NUM_MC_X_NC_BLOCKS ]; + +#define BLIS_USE_HEAP + +#ifdef BLIS_USE_HEAP +static char* pool_mk_mem = NULL; +static char* pool_kn_mem = NULL; +static char* pool_mn_mem = NULL; +#else +static char pool_mk_mem[ BLIS_MK_POOL_SIZE ]; +static char pool_kn_mem[ BLIS_KN_POOL_SIZE ]; +static char pool_mn_mem[ BLIS_MN_POOL_SIZE ]; +#endif + + + +void bli_mem_acquire_m( siz_t req_size, + packbuf_t buf_type, + mem_t* mem ) +{ + siz_t block_size; + dim_t pool_index; + pool_t* pool; + void** block_ptrs; + void* block; + gint_t i; + + + if ( buf_type == BLIS_BUFFER_FOR_GEN_USE ) + { + // For general-use buffer requests, such as those used by level-2 + // operations, using bli_malloc() is sufficient, since using + // physically contiguous memory is not as important there. + block = bli_malloc( req_size ); + + // Initialize the mem_t object with: + // - the address of the memory block, + // - the buffer type (a packbuf_t value), and + // - the size of the requested region. + // NOTE: We do not initialize the pool field since this block did not + // come from a contiguous memory pool. + bli_mem_set_buffer( block, mem ); + bli_mem_set_buf_type( buf_type, mem ); + bli_mem_set_size( req_size, mem ); + } + else + { + // This branch handles cases where the memory block needs to come + // from one of the contiguous memory pools. + + // Map the requested packed buffer type to a zero-based index, which + // we then use to select the corresponding memory pool. + pool_index = bli_packbuf_index( buf_type ); + pool = &pools[ pool_index ]; + + // Unconditionally perform error checking on the memory pool. + { + err_t e_val; + + // Make sure that the requested matrix size fits inside of a block + // of the corresponding pool. + e_val = bli_check_requested_block_size_for_pool( req_size, pool ); + bli_check_error_code( e_val ); + + // Make sure that the pool contains at least one block to check out + // to the thread. + e_val = bli_check_if_exhausted_pool( pool ); + bli_check_error_code( e_val ); + } + + // Access the block pointer array from the memory pool data structure. + block_ptrs = bli_pool_block_ptrs( pool ); + + + // BEGIN CRITICAL SECTION +#ifdef BLIS_ENABLE_OPENMP + _Pragma( "omp critical (mem)" ) +#endif +#ifdef BLIS_ENABLE_PTHREADS + pthread_mutex_lock( &mem_manager_mutex ); +#endif + { + + // Query the index of the contiguous memory block that resides at the + // "top" of the pool. + i = bli_pool_top_index( pool ); + + // Extract the address of the top block from the block pointer array. + block = block_ptrs[i]; + + // Clear the entry from the block pointer array. (This is actually not + // necessary.) + //block_ptrs[i] = NULL; + + // Decrement the top of the memory pool. + bli_pool_dec_top_index( pool ); + + + // END CRITICAL SECTION + } +#ifdef BLIS_ENABLE_PTHREADS + pthread_mutex_unlock( &mem_manager_mutex ); +#endif + + // Query the size of the blocks in the pool so we can store it in the + // mem_t object. + block_size = bli_pool_block_size( pool ); + + // Initialize the mem_t object with: + // - the address of the memory block, + // - the buffer type (a packbuf_t value), + // - the address of the memory pool to which it belongs, and + // - the size of the contiguous memory block (NOT the size of the + // requested region). + bli_mem_set_buffer( block, mem ); + bli_mem_set_buf_type( buf_type, mem ); + bli_mem_set_pool( pool, mem ); + bli_mem_set_size( block_size, mem ); + } +} + + +void bli_mem_release( mem_t* mem ) +{ + packbuf_t buf_type; + pool_t* pool; + void** block_ptrs; + void* block; + gint_t i; + + // Extract the address of the memory block we are trying to + // release. + block = bli_mem_buffer( mem ); + + // Extract the buffer type so we know what kind of memory was allocated. + buf_type = bli_mem_buf_type( mem ); + + if ( buf_type == BLIS_BUFFER_FOR_GEN_USE ) + { + // For general-use buffers, we allocate with bli_malloc(), and so + // here we need to call bli_free(). + bli_free( block ); + } + else + { + // This branch handles cases where the memory block came from one + // of the contiguous memory pools. + + // Extract the pool from which the block was allocated. + pool = bli_mem_pool( mem ); + + // Extract the block pointer array associated with the pool. + block_ptrs = bli_pool_block_ptrs( pool ); + + + // BEGIN CRITICAL SECTION +#ifdef BLIS_ENABLE_OPENMP + _Pragma( "omp critical (mem)" ) +#endif +#ifdef BLIS_ENABLE_PTHREADS + pthread_mutex_lock( &mem_manager_mutex ); +#endif + { + + // Increment the top of the memory pool. + bli_pool_inc_top_index( pool ); + + // Query the newly incremented top index. + i = bli_pool_top_index( pool ); + + // Place the address of the block back onto the top of the memory pool. + block_ptrs[i] = block; + + + // END CRITICAL SECTION + } +#ifdef BLIS_ENABLE_PTHREADS + pthread_mutex_unlock( &mem_manager_mutex ); +#endif + } + + + // Clear the mem_t object so that it appears unallocated. We clear: + // - the buffer field, + // - the pool field, and + // - the size field. + // NOTE: We do not clear the buf_type field since there is no + // "uninitialized" value for packbuf_t. + bli_mem_set_buffer( NULL, mem ); + bli_mem_set_pool( NULL, mem ); + bli_mem_set_size( 0, mem ); +} + + +void bli_mem_acquire_v( siz_t req_size, + mem_t* mem ) +{ + bli_mem_acquire_m( req_size, + BLIS_BUFFER_FOR_GEN_USE, + mem ); +} + + + +void bli_mem_init() +{ + dim_t index_a; + dim_t index_b; + dim_t index_c; + +#ifdef BLIS_USE_HEAP + pool_mk_mem = bli_malloc( BLIS_MK_POOL_SIZE ); + pool_kn_mem = bli_malloc( BLIS_KN_POOL_SIZE ); + pool_mn_mem = bli_malloc( BLIS_MN_POOL_SIZE ); +#endif + + // Map each of the packbuf_t values to an index starting at zero. + index_a = bli_packbuf_index( BLIS_BUFFER_FOR_A_BLOCK ); + index_b = bli_packbuf_index( BLIS_BUFFER_FOR_B_PANEL ); + index_c = bli_packbuf_index( BLIS_BUFFER_FOR_C_PANEL ); + + // Initialize contiguous memory pool for MC x KC blocks. + bli_mem_init_pool( pool_mk_mem, + BLIS_MK_BLOCK_SIZE, + BLIS_NUM_MC_X_KC_BLOCKS, + pool_mk_blk_ptrs, + &pools[ index_a ] ); + + // Initialize contiguous memory pool for KC x NC blocks. + bli_mem_init_pool( pool_kn_mem, + BLIS_KN_BLOCK_SIZE, + BLIS_NUM_KC_X_NC_BLOCKS, + pool_kn_blk_ptrs, + &pools[ index_b ] ); + + // Initialize contiguous memory pool for MC x NC blocks. + bli_mem_init_pool( pool_mn_mem, + BLIS_MN_BLOCK_SIZE, + BLIS_NUM_MC_X_NC_BLOCKS, + pool_mn_blk_ptrs, + &pools[ index_c ] ); +} + + +void bli_mem_init_pool( char* pool_mem, + siz_t block_size, + dim_t num_blocks, + void** block_ptrs, + pool_t* pool ) +{ + const siz_t align_size = BLIS_CONTIG_ADDR_ALIGN_SIZE; + dim_t i; + + // If the pool starting address is not already aligned, advance it + // accordingly. + if ( bli_is_unaligned_to( ( uintptr_t )pool_mem, ( uintptr_t )align_size ) ) + { + // Notice that this works even if the alignment is not a power of two. + pool_mem += ( ( uintptr_t )align_size - + ( ( uintptr_t )pool_mem % align_size ) ); + } + + // Step through the memory pool, beginning with the aligned address + // determined above, assigning pointers to the beginning of each block_size + // bytes to the ith element of the block_ptrs array. + for ( i = 0; i < num_blocks; ++i ) + { + // Save the address of pool, which is guaranteed to be aligned. + block_ptrs[i] = pool_mem; + + // Advance pool by one block. + pool_mem += block_size; + + // Advance pool a bit further if needed in order to get to the + // beginning of an alignment boundary. + if ( bli_is_unaligned_to( ( uintptr_t )pool_mem, ( uintptr_t )align_size ) ) + { + pool_mem += ( ( uintptr_t )align_size - + ( ( uintptr_t )pool_mem % align_size ) ); + } + } + + // Now that we have initialized the array of pointers to the individual + // blocks in the pool, we initialize a pool_t data structure so that we + // can easily manage this pool. + bli_pool_init( num_blocks, + block_size, + block_ptrs, + pool ); +} + + + +void bli_mem_finalize() +{ + // Nothing to do. + +#ifdef BLIS_USE_HEAP + bli_free( pool_mk_mem ); + bli_free( pool_kn_mem ); + bli_free( pool_mn_mem ); +#endif + +} + diff --git a/frame/base/bli_mem.h b/frame/base/bli_mem.h index 6e83ac1223..f80e73aa61 100644 --- a/frame/base/bli_mem.h +++ b/frame/base/bli_mem.h @@ -32,6 +32,13 @@ */ +void bli_mem_init( void ); +void bli_mem_reinit( void ); +void bli_mem_finalize( void ); +bool_t bli_mem_is_initialized( void ); + +// ----------------------------------------------------------------------------- + void bli_mem_acquire_m( siz_t req_size, packbuf_t buf_type, mem_t* mem ); @@ -41,12 +48,20 @@ void bli_mem_acquire_v( siz_t req_size, void bli_mem_release( mem_t* mem ); -void bli_mem_init( void ); -void bli_mem_finalize( void ); +siz_t bli_mem_pool_size( packbuf_t buf_type ); + +// ----------------------------------------------------------------------------- + +void bli_mem_init_pools( void ); +void bli_mem_reinit_pools( void ); +void bli_mem_finalize_pools( void ); -void bli_mem_init_pool( char* pool_mem, - siz_t block_size, - dim_t n_blocks, - void** block_ptrs, - pool_t* pool_struct ); +void bli_mem_compute_pool_block_sizes( siz_t* bs_a, + siz_t* bs_b, + siz_t* bs_c ); +void bli_mem_compute_pool_block_sizes_dt( num_t dt, + ind_t method, + siz_t* bs_a, + siz_t* bs_b, + siz_t* bs_c ); diff --git a/frame/base/bli_obj.c b/frame/base/bli_obj.c index b6277dcb3e..d8d1520031 100644 --- a/frame/base/bli_obj.c +++ b/frame/base/bli_obj.c @@ -117,32 +117,24 @@ void bli_obj_alloc_buffer( inc_t rs, { dim_t n_elem = 0; dim_t m, n; - inc_t rs_abs, cs_abs, is_abs; siz_t elem_size; siz_t buffer_size; void* p; - m = bli_obj_length( *obj ); - n = bli_obj_width( *obj ); + // Query the dimensions of the object we are allocating. + m = bli_obj_length( *obj ); + n = bli_obj_width( *obj ); + + // Query the size of one element. + elem_size = bli_obj_elem_size( *obj ); // Adjust the strides, if needed, before doing anything else // (particularly, before doing any error checking). - bli_adjust_strides( m, n, &rs, &cs, &is ); + bli_adjust_strides( m, n, elem_size, &rs, &cs, &is ); if ( bli_error_checking_is_enabled() ) bli_obj_alloc_buffer_check( rs, cs, is, obj ); - // Query the size of one element. - elem_size = bli_obj_elem_size( *obj ); - - // Compute the magnitude of the row, column, and imaginary strides. - // We will use these in the comparisons below since those - // comparisions really relate only to the magnitudes of the strides, - // not their signs. - rs_abs = bli_abs( rs ); - cs_abs = bli_abs( cs ); - is_abs = bli_abs( is ); - // Determine how much object to allocate. if ( m == 0 || n == 0 ) { @@ -150,44 +142,13 @@ void bli_obj_alloc_buffer( inc_t rs, // should remain unchanged (because alignment is not needed). n_elem = 0; } - else if ( rs_abs == 1 ) - { - cs = bli_align_dim_to_size( cs, elem_size, - BLIS_HEAP_STRIDE_ALIGN_SIZE ); - n_elem = bli_abs( cs ) * n; - } - else if ( cs_abs == 1 ) - { - rs = bli_align_dim_to_size( rs, elem_size, - BLIS_HEAP_STRIDE_ALIGN_SIZE ); - n_elem = bli_abs( rs ) * m; - } else { - if ( rs_abs < cs_abs ) - { - // Note this case is identical to that of rs == 1 above. - cs = bli_align_dim_to_size( cs, elem_size, - BLIS_HEAP_STRIDE_ALIGN_SIZE ); - n_elem = bli_abs( cs ) * n; - } - else if ( cs_abs < rs_abs ) - { - // Note this case is identical to that of cs == 1 above. - rs = bli_align_dim_to_size( rs, elem_size, - BLIS_HEAP_STRIDE_ALIGN_SIZE ); - n_elem = bli_abs( rs ) * m; - } - else // if ( cs_abs == rs_abs ) - { - // If the row and column strides are equal, it almost certainly - // means that m == n == 1. (If that is not the case, then - // something is very wrong.) - if ( m != 1 || m != 1 ) - bli_check_error_code( BLIS_NOT_YET_IMPLEMENTED ); - - n_elem = 1; - } + // The number of elements to allocate is given by the distance from + // the element with the lowest address (usually {0, 0}) to the element + // with the highest address (usually {m-1, n-1}), plus one for the + // highest element itself. + n_elem = (m-1) * bli_abs( rs ) + (n-1) * bli_abs( cs ) + 1; } // Handle the special case where imaginary stride is larger than @@ -197,10 +158,7 @@ void bli_obj_alloc_buffer( inc_t rs, // Notice that adding is/2 works regardless of whether the // imaginary stride is unit, something between unit and // 2*n_elem, or something bigger than 2*n_elem. - if ( 1 > is_abs ) - { - n_elem = is_abs / 2 + n_elem; - } + n_elem = bli_abs( is ) / 2 + n_elem; } // Compute the size of the total buffer to be allocated, which includes @@ -222,16 +180,9 @@ void bli_obj_attach_buffer( void* p, inc_t is, obj_t* obj ) { - dim_t m, n; - - m = bli_obj_length( *obj ); - n = bli_obj_width( *obj ); - - // Adjust the strides, if necessary. - bli_adjust_strides( m, n, &rs, &cs, &is ); - - // Notice that we wait until after strides have been adjusted to - // error-check. + // Check that the strides and lengths are compatible. Note that the + // user *must* specify valid row and column strides when attaching an + // external buffer. if ( bli_error_checking_is_enabled() ) bli_obj_attach_buffer_check( p, rs, cs, is, obj ); @@ -359,6 +310,7 @@ void bli_obj_create_const_copy_of( obj_t* a, obj_t* b ) void bli_adjust_strides( dim_t m, dim_t n, + siz_t elem_size, inc_t* rs, inc_t* cs, inc_t* is ) @@ -367,16 +319,10 @@ void bli_adjust_strides( dim_t m, // them if needed. // Handle the special "empty" case first. If either dimension is zero, - // we set row and column strides to zero. - if ( m == 0 || n == 0 ) - { - *rs = 0; - *cs = 0; - *is = 1; + // do nothing (this could represent a zero-length "slice" of another + // matrix). + if ( m == 0 || n == 0 ) return; - return; - } - // Interpret rs = cs = 0 as request for column storage. if ( *rs == 0 && *cs == 0 && ( *is == 0 || *is == 1 ) ) { @@ -401,6 +347,21 @@ void bli_adjust_strides( dim_t m, // Use default complex storage. *is = 1; + + // Align the strides depending on the tilt of the matrix. Note that + // scalars are neither row nor column tilted. Also note that alignment + // is only done for rs = cs = 0, and any user-supplied row and column + // strides are preserved. + if ( bli_is_col_tilted( m, n, *rs, *cs ) ) + { + *cs = bli_align_dim_to_size( *cs, elem_size, + BLIS_HEAP_STRIDE_ALIGN_SIZE ); + } + else if ( bli_is_row_tilted( m, n, *rs, *cs ) ) + { + *rs = bli_align_dim_to_size( *rs, elem_size, + BLIS_HEAP_STRIDE_ALIGN_SIZE ); + } } else if ( *rs == 1 && *cs == 1 ) { @@ -509,8 +470,8 @@ void bli_obj_print( char* label, obj_t* obj ) ( unsigned long int )bli_obj_width( *obj ) ); fprintf( file, "\n" ); - fprintf( file, " offm, offn %lu, %lu\n", ( unsigned long int )bli_obj_row_offset( *obj ), - ( unsigned long int )bli_obj_col_offset( *obj ) ); + fprintf( file, " offm, offn %lu, %lu\n", ( unsigned long int )bli_obj_row_off( *obj ), + ( unsigned long int )bli_obj_col_off( *obj ) ); fprintf( file, " diagoff %ld\n", ( signed long int )bli_obj_diag_offset( *obj ) ); fprintf( file, "\n" ); diff --git a/frame/base/bli_obj.h b/frame/base/bli_obj.h index 316024012e..92e9b1d874 100644 --- a/frame/base/bli_obj.h +++ b/frame/base/bli_obj.h @@ -80,6 +80,7 @@ void bli_obj_create_const_copy_of( obj_t* a, obj_t* b ); void bli_adjust_strides( dim_t m, dim_t n, + siz_t elem_size, inc_t* rs, inc_t* cs, inc_t* is ); diff --git a/frame/base/bli_part.c b/frame/base/bli_part.c index b8a28d5415..64718353eb 100644 --- a/frame/base/bli_part.c +++ b/frame/base/bli_part.c @@ -592,6 +592,16 @@ void bli_acquire_mpart_tl2br( subpart_t requested_part, requested_part != BLIS_SUBPART11 && requested_part != BLIS_SUBPART22 ) { + // FGVZ: Fix me. This needs to be cleaned up. Either non-diagonal + // intersecting subpartitions should inherit their root object's + // uplo field, or it should not. Right now, they DO inherit the + // uplo (because they are not set to BLIS_DENSE when the diagonal + // does not intersect). But the whole point of being able to query + // the root object's properties (e.g. uplo field) was so that we + // COULD mark such subpartitions as dense, to make it easier for + // certain subproblems on those subpartitions--subproblems that + // are agnostic to where the subpartition came from. + // NOTE: This comment may be out-of-date since we now distinguish // between uplo properties for the current and root objects... // Note that we cannot mark the subpartition object as general/dense diff --git a/frame/base/bli_pool.c b/frame/base/bli_pool.c new file mode 100644 index 0000000000..bae3e3556f --- /dev/null +++ b/frame/base/bli_pool.c @@ -0,0 +1,385 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + +void bli_pool_init( dim_t num_blocks_init, + siz_t block_size, + siz_t align_size, + pool_t* pool ) +{ + pblk_t* block_ptrs; + dim_t i; + + // Allocate the block_ptrs array. + block_ptrs = bli_malloc( num_blocks_init * sizeof( pblk_t ) ); + + // Allocate and initialize each entry in the block_ptrs array. + for ( i = 0; i < num_blocks_init; ++i ) + { + bli_pool_alloc_block( block_size, align_size, &(block_ptrs[i]) ); + } + + // Initialize the pool_t structure. + bli_pool_set_block_ptrs( block_ptrs, pool ); + bli_pool_set_block_ptrs_len( num_blocks_init, pool ); + bli_pool_set_num_blocks( num_blocks_init, pool ); + bli_pool_set_top_index( 0, pool ); + bli_pool_set_block_size( block_size, pool ); + bli_pool_set_align_size( align_size, pool ); +} + +void bli_pool_finalize( pool_t* pool ) +{ + pblk_t* block_ptrs; + dim_t num_blocks; + dim_t top_index; + dim_t i; + + // NOTE: This implementation assumes that either: + // - all blocks have been checked in by all threads, or + // - some subset of blocks have been checked in and the caller + // is bli_pool_reinit(). + + // Query the current block_ptrs array. + block_ptrs = bli_pool_block_ptrs( pool ); + + // Query the total number of blocks presently allocated. + num_blocks = bli_pool_num_blocks( pool ); + + // Query the top_index of the pool. + top_index = bli_pool_top_index( pool ); + + // Free the individual blocks currently in the pool. + for ( i = top_index; i < num_blocks; ++i ) + { + bli_pool_free_block( &(block_ptrs[i]) ); + } + + // Free the block_ptrs array. + bli_free( block_ptrs ); + + // Clear the contents of the pool_t struct. + bli_pool_set_block_ptrs( NULL, pool ); + bli_pool_set_block_ptrs_len( 0, pool ); + bli_pool_set_num_blocks( 0, pool ); + bli_pool_set_top_index( 0, pool ); + bli_pool_set_block_size( 0, pool ); + bli_pool_set_align_size( 0, pool ); +} + +void bli_pool_reinit( dim_t num_blocks_new, + siz_t block_size_new, + siz_t align_size_new, + pool_t* pool ) +{ + // Finalize the pool as it is currently configured. If some blocks + // are still checked out to threads, those blocks are not freed + // here, and instead will be freed when the threads are ready to + // release the blocks. (This will happen because the threads will + // notice that the block size of the pool has changed.) + bli_pool_finalize( pool ); + + // Reinitialize the pool with the new parameters, in particular, + // the new block size. + bli_pool_init( num_blocks_new, block_size_new, align_size_new, pool ); +} + +void bli_pool_checkout_block( pblk_t* block, pool_t* pool ) +{ + pblk_t* block_ptrs; + dim_t top_index; + + // If the pool is exhausted, add a block. + if ( bli_pool_is_exhausted( pool ) ) + { + bli_pool_grow( 1, pool ); + } + + // At this point, at least one block is guaranteed to be available. + + // Query the current block_ptrs array. + block_ptrs = bli_pool_block_ptrs( pool ); + + // Query the top_index of the pool. + top_index = bli_pool_top_index( pool ); + + // Copy the block at top_index to the caller's pblk_t struct. + //bli_pblk_copy( *(block_ptrs[top_index]), *block ); + *block = block_ptrs[top_index]; + + // Notice that we don't actually need to clear the contents of + // block_ptrs[top_index]. It will get overwritten eventually when + // the block is checked back in. + bli_pblk_clear( &block_ptrs[top_index] ); + + // Increment the pool's top_index. + bli_pool_set_top_index( top_index + 1, pool ); +} + +void bli_pool_checkin_block( pblk_t* block, pool_t* pool ) +{ + pblk_t* block_ptrs; + dim_t top_index; + + // Query the current block_ptrs array. + block_ptrs = bli_pool_block_ptrs( pool ); + + // Query the top_index of the pool. + top_index = bli_pool_top_index( pool ); + + // Copy the caller's pblk_t struct to the block at top_index - 1. + //bli_pblk_copy( *(block_ptrs[top_index-1]), *block ); + block_ptrs[top_index-1] = *block; + + // Decrement the pool's top_index. + bli_pool_set_top_index( top_index - 1, pool ); +} + +void bli_pool_grow( dim_t num_blocks_add, pool_t* pool ) +{ + pblk_t* block_ptrs_cur; + dim_t block_ptrs_len_cur; + dim_t num_blocks_cur; + + pblk_t* block_ptrs_new; + dim_t num_blocks_new; + + siz_t block_size; + siz_t align_size; + dim_t top_index; + + dim_t i; + + // If the requested increase is zero (or negative), return early. + if ( num_blocks_add < 1 ) return; + + // Query the allocated length of the block_ptrs array and also the + // total number of blocks allocated. + block_ptrs_len_cur = bli_pool_block_ptrs_len( pool ); + num_blocks_cur = bli_pool_num_blocks( pool ); + + // Compute the total number of allocated blocks that will exist + // after we grow the pool. + num_blocks_new = num_blocks_cur + num_blocks_add; + + // If the new total number of allocated blocks is larger than the + // allocated length of the block_ptrs array, we need to allocate + // a new (larger) block_ptrs array. + if ( num_blocks_new > block_ptrs_len_cur ) + { + // Query the current block_ptrs array. + block_ptrs_cur = bli_pool_block_ptrs( pool ); + + // Allocate a new block_ptrs array of length num_blocks_new. + block_ptrs_new = bli_malloc( num_blocks_new * sizeof( pblk_t ) ); + + // Query the top_index of the pool. + top_index = bli_pool_top_index( pool ); + + // Copy the contents of the old block_ptrs array to the new/resized + // array. Notice that we can begin with top_index since all entries + // from 0 to top_index-1 have been checked out to threads. + for ( i = top_index; i < num_blocks_cur; ++i ) + { +//printf( "bli_pool_grow: copying from %lu\n", top_index ); + block_ptrs_new[i] = block_ptrs_cur[i]; + } + +//printf( "bli_pool_grow: bp_cur: %p\n", block_ptrs_cur ); + // Free the old block_ptrs array. + bli_free( block_ptrs_cur ); + + // Update the pool_t struct with the new block_ptrs array and + // record its allocated length. + bli_pool_set_block_ptrs( block_ptrs_new, pool ); + bli_pool_set_block_ptrs_len( num_blocks_new, pool ); + } + + // At this point, we are guaranteed to have enough unused elements + // in the block_ptrs array to accommodate an additional num_blocks_add + // blocks. + + // Query the current block_ptrs array (which was possibly just resized). + block_ptrs_cur = bli_pool_block_ptrs( pool ); + + // Query the block size and alignment size of the current pool. + block_size = bli_pool_block_size( pool ); + align_size = bli_pool_align_size( pool ); + + // Allocate the requested additional blocks in the resized array. + for ( i = num_blocks_cur; i < num_blocks_new; ++i ) + { +//printf( "libblis: growing pool, block_size = %lu\n", block_size ); fflush( stdout ); + + bli_pool_alloc_block( block_size, align_size, &(block_ptrs_cur[i]) ); + } + + // Update the pool_t struct with the new number of allocated blocks. + // Notice that top_index remains unchanged, as do the block_size and + // align_size fields. + bli_pool_set_num_blocks( num_blocks_new, pool ); +} + +void bli_pool_shrink( dim_t num_blocks_sub, pool_t* pool ) +{ + pblk_t* block_ptrs; + dim_t num_blocks; + dim_t num_blocks_avail; + dim_t num_blocks_new; + + dim_t top_index; + + dim_t i; + + // Query the total number of blocks presently allocated. + num_blocks = bli_pool_num_blocks( pool ); + + // Query the top_index of the pool. + top_index = bli_pool_top_index( pool ); + + // Compute the number of blocks available to be checked out + // (and thus available for removal). + num_blocks_avail = num_blocks - top_index; + + // If the requested decrease is more than the number of available + // blocks in the pool, only remove the number of blocks available. + if ( num_blocks_avail < num_blocks_sub ) + num_blocks_sub = num_blocks_avail; + + // If the effective requested decrease is zero (or the requested + // decrease was negative), return early. + if ( num_blocks_sub < 1 ) return; + + // Query the current block_ptrs array. + block_ptrs = bli_pool_block_ptrs( pool ); + + // Compute the new total number of blocks. + num_blocks_new = num_blocks - num_blocks_sub; + + // Free the individual blocks. + for ( i = num_blocks_new; i < num_blocks; ++i ) + { + bli_pool_free_block( &(block_ptrs[i]) ); + } + + // Update the pool_t struct. + bli_pool_set_num_blocks( num_blocks_new, pool ); + + // Note that after shrinking the pool, num_blocks < block_ptrs_len. + // This means the pool can grow again by num_blocks_sub before + // a re-allocation of block_ptrs is triggered. +} + +void bli_pool_alloc_block( siz_t block_size, + siz_t align_size, + pblk_t* block ) +{ + void* buf_sys; + void* buf_align; + + // Allocate the block. We add the alignment size to ensure we will + // have enough usable space after alignment. + buf_sys = bli_malloc( block_size + align_size ); + buf_align = buf_sys; + + // Advance the pointer to achieve the necessary alignment, if it is not + // already aligned. + if ( bli_is_unaligned_to( buf_sys, align_size ) ) + { + // C99's stdint.h guarantees that a void* can be safely cast to a + // uintptr_t and then back to a void*, hence the casting of buf_sys + // and align_size to uintptr_t. buf_align is initially cast to char* + // to allow pointer arithmetic in units of bytes, and then advanced + // to the next nearest alignment boundary, and finally cast back to + // void* before being stored. Notice that the arithmetic works even + // if the alignment value is not a power of two. + buf_align = ( void* )( ( char* )buf_align + + ( ( uintptr_t )align_size - + ( uintptr_t )buf_sys % + ( uintptr_t )align_size ) + ); + } + + // Save the results in the pblk_t structure. + bli_pblk_set_buf_sys( buf_sys, block ); + bli_pblk_set_buf_align( buf_align, block ); +} + +void bli_pool_free_block( pblk_t* block ) +{ + void* buf_sys; + + // Extract the pointer to the block that was originally provided by + // the operating system. + buf_sys = bli_pblk_buf_sys( block ); + + // Free the block. + bli_free( buf_sys ); +} + +void bli_pool_print( pool_t* pool ) +{ + pblk_t* block_ptrs = bli_pool_block_ptrs( pool ); + dim_t block_ptrs_len = bli_pool_block_ptrs_len( pool ); + dim_t top_index = bli_pool_top_index( pool ); + dim_t num_blocks = bli_pool_num_blocks( pool ); + dim_t block_size = bli_pool_block_size( pool ); + dim_t align_size = bli_pool_align_size( pool ); + dim_t i; + + printf( "pool struct ---------------\n" ); + printf( " block_ptrs: %p\n", block_ptrs ); + printf( " block_ptrs_len: %ld\n", block_ptrs_len ); + printf( " top_index: %ld\n", top_index ); + printf( " num_blocks: %ld\n", num_blocks ); + printf( " block_size: %ld\n", block_size ); + printf( " align_size: %ld\n", align_size ); + printf( " pblks sys align\n" ); + for ( i = 0; i < num_blocks; ++i ) + { + printf( " %ld: %p %p\n", i, bli_pblk_buf_sys( &block_ptrs[i] ), + bli_pblk_buf_align( &block_ptrs[i] ) ); + } +} + +void bli_pblk_print( pblk_t* pblk ) +{ + void* buf_sys = bli_pblk_buf_sys( pblk ); + void* buf_align = bli_pblk_buf_align( pblk ); + + printf( "pblk struct ---------------\n" ); + printf( " %p %p\n", buf_sys, buf_align ); +} + diff --git a/frame/base/bli_pool.h b/frame/base/bli_pool.h new file mode 100644 index 0000000000..aba3c2f51c --- /dev/null +++ b/frame/base/bli_pool.h @@ -0,0 +1,184 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef BLIS_POOL_H +#define BLIS_POOL_H + +// -- Pool block type -- + +/* +typedef struct +{ + void* buf_sys; + void* buf_align; +} pblk_t; +*/ + +// -- Pool type -- + +/* +typedef struct +{ + pblk_t* block_ptrs; + dim_t block_ptrs_len; + + dim_t top_index; + dim_t num_blocks; + + siz_t block_size; + siz_t align_size; +} pool_t; +*/ + + +// Pool block query + +#define bli_pblk_buf_sys( pblk_p ) \ +\ + ( (pblk_p)->buf_sys ) + +#define bli_pblk_buf_align( pblk_p ) \ +\ + ( (pblk_p)->buf_align ) + +// Pool block modification + +#define bli_pblk_set_buf_sys( buf_sys0, pblk_p ) \ +{ \ + (pblk_p)->buf_sys = buf_sys0; \ +} + +#define bli_pblk_set_buf_align( buf_align0, pblk_p ) \ +{ \ + (pblk_p)->buf_align = buf_align0; \ +} + +#define bli_pblk_clear( pblk_p ) \ +{ \ + bli_pblk_set_buf_sys( NULL, pblk_p ); \ + bli_pblk_set_buf_align( NULL, pblk_p ); \ +} + + +// Pool entry query + +#define bli_pool_block_ptrs( pool_p ) \ +\ + ( (pool_p)->block_ptrs ) + +#define bli_pool_block_ptrs_len( pool_p ) \ +\ + ( (pool_p)->block_ptrs_len ) + +#define bli_pool_num_blocks( pool_p ) \ +\ + ( (pool_p)->num_blocks ) + +#define bli_pool_block_size( pool_p ) \ +\ + ( (pool_p)->block_size ) + +#define bli_pool_align_size( pool_p ) \ +\ + ( (pool_p)->align_size ) + +#define bli_pool_top_index( pool_p ) \ +\ + ( (pool_p)->top_index ) + +#define bli_pool_is_exhausted( pool_p ) \ +\ + ( bli_pool_top_index( pool_p ) == \ + bli_pool_num_blocks( pool_p ) ) + +// Pool entry modification + +#define bli_pool_set_block_ptrs( block_ptrs0, pool_p ) \ +{ \ + (pool_p)->block_ptrs = block_ptrs0; \ +} + +#define bli_pool_set_block_ptrs_len( block_ptrs_len0, pool_p ) \ +{ \ + (pool_p)->block_ptrs_len = block_ptrs_len0; \ +} + +#define bli_pool_set_num_blocks( num_blocks0, pool_p ) \ +{ \ + (pool_p)->num_blocks = num_blocks0; \ +} + +#define bli_pool_set_block_size( block_size0, pool_p ) \ +{ \ + (pool_p)->block_size = block_size0; \ +} + +#define bli_pool_set_align_size( align_size0, pool_p ) \ +{ \ + (pool_p)->align_size = align_size0; \ +} + +#define bli_pool_set_top_index( top_index0, pool_p ) \ +{ \ + (pool_p)->top_index = top_index0; \ +} + +#endif + +// ----------------------------------------------------------------------------- + +void bli_pool_init( dim_t num_blocks_init, + siz_t block_size, + siz_t align_size, + pool_t* pool ); +void bli_pool_finalize( pool_t* pool ); +void bli_pool_reinit( dim_t num_blocks_new, + siz_t block_size, + siz_t align_size, + pool_t* pool ); + +void bli_pool_checkout_block( pblk_t* block, pool_t* pool ); +void bli_pool_checkin_block( pblk_t* block, pool_t* pool ); + +void bli_pool_grow( dim_t num_blocks_add, pool_t* pool ); +void bli_pool_shrink( dim_t num_blocks_sub, pool_t* pool ); + +void bli_pool_alloc_block( siz_t block_size, + siz_t align_size, + pblk_t* block ); +void bli_pool_free_block( pblk_t* block ); + +void bli_pool_print( pool_t* pool ); +void bli_pblk_print( pblk_t* pblk ); + diff --git a/frame/base/bli_prune.c b/frame/base/bli_prune.c new file mode 100644 index 0000000000..d53049ecc3 --- /dev/null +++ b/frame/base/bli_prune.c @@ -0,0 +1,135 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + +void bli_prune_unref_mparts( obj_t* p, mdim_t mdim_p, + obj_t* s, mdim_t mdim_s ) +{ + // If the primary object is general, it has no structure, and + // therefore, no unreferenced parts. + if ( bli_obj_is_general( *p ) ) return; + + // If the primary object is BLIS_ZEROS, set the dimensions so that the + // matrix is empty. This is not strictly needed but rather a minor + // optimization, as it would prevent threads that would otherwise get + // subproblems on BLIS_ZEROS operands from calling the macro-kernel, + // because bli_get_range*() would return empty ranges, which would + // cause the variant's for loop from executing any iterations. + // NOTE: this should only ever execute if the primary object is + // triangular because that is the only structure type with subpartitions + // that can be marked as BLIS_ZEROS. + if ( bli_obj_is_triangular( *p ) && + bli_obj_is_zeros( *p ) ) { bli_obj_set_dim( mdim_p, 0, *p ); + bli_obj_set_dim( mdim_s, 0, *s ); + return; } + + // If the primary object is hermitian, symmetric, or triangular, we + // assume that the unstored region will be unreferenced (otherwise, + // the caller should not be invoking this function on that object). + //if ( bli_obj_is_herm_or_symm( *p ) || + // bli_obj_is_triangular( *p ) ) + { + doff_t diagoff_p = bli_obj_diag_offset( *p ); + dim_t m = bli_obj_length( *p ); + dim_t n = bli_obj_width( *p ); + uplo_t uplo = bli_obj_uplo( *p ); + dim_t off_inc = 0; + dim_t q; + + // Support implicit transposition on p and s. + if ( bli_obj_has_trans( *p ) ) + { + bli_reflect_about_diag( diagoff_p, uplo, m, n ); + bli_toggle_dim( mdim_p ); + } + if ( bli_obj_has_trans( *s ) ) + { + bli_toggle_dim( mdim_s ); + } + + // Prune away any zero region of the matrix depending on the + // dimension of the primary object being partitioned and the + // triangle in which it is stored. + if ( bli_obj_is_lower( *p ) ) + { + if ( bli_is_m_dim( mdim_p ) ) + { bli_prune_unstored_region_top_l( diagoff_p, m, n, off_inc ); } + else // if ( bli_is_n_dim( mdim_p ) ) + { bli_prune_unstored_region_right_l( diagoff_p, m, n, off_inc ); } + } + else if ( bli_obj_is_upper( *p ) ) + { + if ( bli_is_m_dim( mdim_p ) ) + { bli_prune_unstored_region_bottom_u( diagoff_p, m, n, off_inc ); } + else // if ( bli_is_n_dim( mdim_p ) ) + { bli_prune_unstored_region_left_u( diagoff_p, m, n, off_inc ); } + } + else if ( bli_obj_is_dense( *p ) ) + { + // Hermitian, symmetric, and triangular matrices are almost + // never dense, but if one were found to be dense, it would + // have no unreferenced regions to prune. + return; + } + else // if ( bli_obj_is_zeros( *p ) ) + { + // Sanity check. Hermitian/symmetric matrices should never have + // zero subpartitions. + bli_check_error_code( BLIS_NOT_YET_IMPLEMENTED ); + } + + // Select the (potentially modified) dimension along which we are + // partitioning. + if ( bli_is_m_dim( mdim_p ) ) q = m; + else /* if ( bli_is_n_dim( mdim_p ) ) */ q = n; + + // Update the affected objects in case anything changed. Notice that + // it is okay to update the dimension and diagonal offset fields of + // packed primary objects, as long as we do so in tandem with the + // secondary object to maintain conformality. This just means that + // the "ignore-able" zero region is skipped over here, rather than + // within the macro-kernel. + bli_obj_set_diag_offset( diagoff_p, *p ); + bli_obj_set_dim( mdim_p, q, *p ); + bli_obj_set_dim( mdim_s, q, *s ); + + // Only update the affected offset fields if the object in question + // is NOT a packed object. Otherwise, bli_obj_buffer_at_off() will + // compute the wrong address within the macro-kernel object wrapper. + if ( !bli_obj_is_packed( *p ) ) { bli_obj_inc_off( mdim_p, off_inc, *p ); } + if ( !bli_obj_is_packed( *s ) ) { bli_obj_inc_off( mdim_s, off_inc, *s ); } + } +} + diff --git a/frame/base/bli_prune.h b/frame/base/bli_prune.h new file mode 100644 index 0000000000..1f71610c80 --- /dev/null +++ b/frame/base/bli_prune.h @@ -0,0 +1,36 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +void bli_prune_unref_mparts( obj_t* p, mdim_t mdim_p, + obj_t* s, mdim_t mdim_s ); diff --git a/frame/base/bli_threading.c b/frame/base/bli_threading.c index 65ef8e6a36..168679ef0f 100644 --- a/frame/base/bli_threading.c +++ b/frame/base/bli_threading.c @@ -34,6 +34,40 @@ #include "blis.h" +static bool_t bli_thread_is_init = FALSE; + +packm_thrinfo_t BLIS_PACKM_SINGLE_THREADED; +gemm_thrinfo_t BLIS_GEMM_SINGLE_THREADED; +herk_thrinfo_t BLIS_HERK_SINGLE_THREADED; +thread_comm_t BLIS_SINGLE_COMM; + +void bli_thread_init( void ) +{ + // If the API is already initialized, return early. + if ( bli_thread_is_initialized() ) return; + + bli_setup_communicator( &BLIS_SINGLE_COMM, 1 ); + bli_setup_packm_single_threaded_info( &BLIS_PACKM_SINGLE_THREADED ); + bli_setup_gemm_single_threaded_info( &BLIS_GEMM_SINGLE_THREADED ); + bli_setup_herk_single_threaded_info( &BLIS_HERK_SINGLE_THREADED ); + + // Mark API as initialized. + bli_thread_is_init = TRUE; +} + +void bli_thread_finalize( void ) +{ + // Mark API as uninitialized. + bli_thread_is_init = FALSE; +} + +bool_t bli_thread_is_initialized( void ) +{ + return bli_thread_is_init; +} + +// ----------------------------------------------------------------------------- + //*********** Stuff Specific to single-threaded ************* #ifndef BLIS_ENABLE_MULTITHREADING void bli_barrier( thread_comm_t* communicator, dim_t t_id ) @@ -123,65 +157,656 @@ void* bli_broadcast_structure( thread_comm_t* communicator, dim_t id, void* to_s } // Code for work assignments -void bli_get_range( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, dim_t* start, dim_t* end ) -{ - thrinfo_t* thread = (thrinfo_t*) thr; - dim_t n_way = thread->n_way; - dim_t work_id = thread->work_id; - - dim_t size = all_end - all_start; - dim_t n_pt = size / n_way; - n_pt = (n_pt * n_way < size) ? n_pt + 1 : n_pt; - n_pt = (n_pt % block_factor == 0) ? n_pt : n_pt + block_factor - (n_pt % block_factor); - *start = work_id * n_pt + all_start; - *end = bli_min( *start + n_pt, size + all_start ); -} - -void bli_get_range_weighted( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, bool_t forward, dim_t* start, dim_t* end) -{ - thrinfo_t* thread = (thrinfo_t*) thr; - dim_t n_way = thread->n_way; - dim_t work_id = thread->work_id; - dim_t size = all_end - all_start; - - *start = 0; - *end = all_end - all_start; - double num = size*size / (double) n_way; - - if( forward ) { - dim_t curr_caucus = n_way - 1; - dim_t len = 0; - while(1){ - dim_t width = ceil(sqrt( len*len + num )) - len; // The width of the current caucus - width = (width % block_factor == 0) ? width : width + block_factor - (width % block_factor); - if( curr_caucus == work_id ) { - *start = bli_max( 0 , *end - width ) + all_start; - *end = *end + all_start; - return; - } - else{ - *end -= width; - len += width; - curr_caucus--; - } - } - } - else{ - while(1){ - dim_t width = ceil(sqrt(*start * *start + num)) - *start; - width = (width % block_factor == 0) ? width : width + block_factor - (width % block_factor); - - if( work_id == 0 ) { - *start = *start + all_start; - *end = bli_min( *start + width, all_end ); - return; - } - else{ - *start = *start + width; - } - work_id--; - } - } +void bli_get_range( void* thr, dim_t n, dim_t bf, bool_t handle_edge_low, dim_t* start, dim_t* end ) +{ + thrinfo_t* thread = ( thrinfo_t* )thr; + dim_t n_way = thread->n_way; + dim_t work_id = thread->work_id; + + dim_t all_start = 0; + dim_t all_end = n; + + dim_t size = all_end - all_start; + + dim_t n_bf_whole = size / bf; + dim_t n_bf_left = size % bf; + + dim_t n_bf_lo = n_bf_whole / n_way; + dim_t n_bf_hi = n_bf_whole / n_way; + + // In this function, we partition the space between all_start and + // all_end into n_way partitions, each a multiple of block_factor + // with the exception of the one partition that recieves the + // "edge" case (if applicable). + // + // Here are examples of various thread partitionings, in units of + // the block_factor, when n_way = 4. (A '+' indicates the thread + // that receives the leftover edge case (ie: n_bf_left extra + // rows/columns in its sub-range). + // (all_start ... all_end) + // n_bf_whole _left hel n_th_lo _hi thr0 thr1 thr2 thr3 + // 12 =0 f 0 4 3 3 3 3 + // 12 >0 f 0 4 3 3 3 3+ + // 13 >0 f 1 3 4 3 3 3+ + // 14 >0 f 2 2 4 4 3 3+ + // 15 >0 f 3 1 4 4 4 3+ + // 15 =0 f 3 1 4 4 4 3 + // + // 12 =0 t 4 0 3 3 3 3 + // 12 >0 t 4 0 3+ 3 3 3 + // 13 >0 t 3 1 3+ 3 3 4 + // 14 >0 t 2 2 3+ 3 4 4 + // 15 >0 t 1 3 3+ 4 4 4 + // 15 =0 t 1 3 3 4 4 4 + + // As indicated by the table above, load is balanced as equally + // as possible, even in the presence of an edge case. + + // First, we must differentiate between cases where the leftover + // "edge" case (n_bf_left) should be allocated to a thread partition + // at the low end of the index range or the high end. + + if ( handle_edge_low == FALSE ) + { + // Notice that if all threads receive the same number of + // block_factors, those threads are considered "high" and + // the "low" thread group is empty. + dim_t n_th_lo = n_bf_whole % n_way; + //dim_t n_th_hi = n_way - n_th_lo; + + // If some partitions must have more block_factors than others + // assign the slightly larger partitions to lower index threads. + if ( n_th_lo != 0 ) n_bf_lo += 1; + + // Compute the actual widths (in units of rows/columns) of + // individual threads in the low and high groups. + dim_t size_lo = n_bf_lo * bf; + dim_t size_hi = n_bf_hi * bf; + + // Precompute the starting indices of the low and high groups. + dim_t lo_start = all_start; + dim_t hi_start = all_start + n_th_lo * size_lo; + + // Compute the start and end of individual threads' ranges + // as a function of their work_ids and also the group to which + // they belong (low or high). + if ( work_id < n_th_lo ) + { + *start = lo_start + (work_id ) * size_lo; + *end = lo_start + (work_id+1) * size_lo; + } + else // if ( n_th_lo <= work_id ) + { + *start = hi_start + (work_id-n_th_lo ) * size_hi; + *end = hi_start + (work_id-n_th_lo+1) * size_hi; + + // Since the edge case is being allocated to the high + // end of the index range, we have to advance the last + // thread's end. + if ( work_id == n_way - 1 ) *end += n_bf_left; + } + } + else // if ( handle_edge_low == TRUE ) + { + // Notice that if all threads receive the same number of + // block_factors, those threads are considered "low" and + // the "high" thread group is empty. + dim_t n_th_hi = n_bf_whole % n_way; + dim_t n_th_lo = n_way - n_th_hi; + + // If some partitions must have more block_factors than others + // assign the slightly larger partitions to higher index threads. + if ( n_th_hi != 0 ) n_bf_hi += 1; + + // Compute the actual widths (in units of rows/columns) of + // individual threads in the low and high groups. + dim_t size_lo = n_bf_lo * bf; + dim_t size_hi = n_bf_hi * bf; + + // Precompute the starting indices of the low and high groups. + dim_t lo_start = all_start; + dim_t hi_start = all_start + n_th_lo * size_lo + + n_bf_left; + + // Compute the start and end of individual threads' ranges + // as a function of their work_ids and also the group to which + // they belong (low or high). + if ( work_id < n_th_lo ) + { + *start = lo_start + (work_id ) * size_lo; + *end = lo_start + (work_id+1) * size_lo; + + // Since the edge case is being allocated to the low + // end of the index range, we have to advance the + // starts/ends accordingly. + if ( work_id == 0 ) *end += n_bf_left; + else { *start += n_bf_left; + *end += n_bf_left; } + } + else // if ( n_th_lo <= work_id ) + { + *start = hi_start + (work_id-n_th_lo ) * size_hi; + *end = hi_start + (work_id-n_th_lo+1) * size_hi; + } + } +} + +siz_t bli_get_range_l2r( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ) +{ + dim_t m = bli_obj_length_after_trans( *a ); + dim_t n = bli_obj_width_after_trans( *a ); + + bli_get_range( thr, n, bf, + FALSE, start, end ); + + return m * ( *end - *start ); +} + +siz_t bli_get_range_r2l( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ) +{ + dim_t m = bli_obj_length_after_trans( *a ); + dim_t n = bli_obj_width_after_trans( *a ); + + bli_get_range( thr, n, bf, + TRUE, start, end ); + + return m * ( *end - *start ); +} + +siz_t bli_get_range_t2b( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ) +{ + dim_t m = bli_obj_length_after_trans( *a ); + dim_t n = bli_obj_width_after_trans( *a ); + + bli_get_range( thr, m, bf, + FALSE, start, end ); + + return n * ( *end - *start ); +} + +siz_t bli_get_range_b2t( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ) +{ + dim_t m = bli_obj_length_after_trans( *a ); + dim_t n = bli_obj_width_after_trans( *a ); + + bli_get_range( thr, m, bf, + TRUE, start, end ); + + return n * ( *end - *start ); +} + +dim_t bli_get_range_width_l( doff_t diagoff_j, + dim_t m, + dim_t n_j, + dim_t j, + dim_t n_way, + dim_t bf, + dim_t bf_left, + double area_per_thr, + bool_t handle_edge_low ) +{ + dim_t width; + + // In this function, we assume that we are somewhere in the process of + // partitioning an m x n lower-stored region (with arbitrary diagonal + // offset) n_ways along the n dimension (into column panels). The value + // j identifies the left-to-right subpartition index (from 0 to n_way-1) + // of the subpartition whose width we are about to compute using the + // area per thread determined by the caller. n_j is the number of + // columns in the remaining region of the matrix being partitioned, + // and diagoff_j is that region's diagonal offset. + + // If this is the last subpartition, the width is simply equal to n_j. + // Note that this statement handles cases where the "edge case" (if + // one exists) is assigned to the high end of the index range (ie: + // handle_edge_low == FALSE). + if ( j == n_way - 1 ) return n_j; + + // At this point, we know there are at least two subpartitions left. + // We also know that IF the submatrix contains a completely dense + // rectangular submatrix, it will occur BEFORE the triangular (or + // trapezoidal) part. + + // Here, we implement a somewhat minor load balancing optimization + // that ends up getting employed only for relatively small matrices. + // First, recall that all subpartition widths will be some multiple + // of the blocking factor bf, except perhaps either the first or last + // subpartition, which will receive the edge case, if it exists. + // Also recall that j represents the current thread (or thread group, + // or "caucus") for which we are computing a subpartition width. + // If n_j is sufficiently small that we can only allocate bf columns + // to each of the remaining threads, then we set the width to bf. We + // do not allow the subpartition width to be less than bf, so, under + // some conditions, if n_j is small enough, some of the reamining + // threads may not get any work. For the purposes of this lower bound + // on work (ie: width >= bf), we allow the edge case to count as a + // "full" set of bf columns. + { + dim_t n_j_bf = n_j / bf + ( bf_left > 0 ? 1 : 0 ); + + if ( n_j_bf <= n_way - j ) + { + if ( j == 0 && handle_edge_low ) + width = ( bf_left > 0 ? bf_left : bf ); + else + width = bf; + + // Make sure that the width does not exceed n_j. This would + // occur if and when n_j_bf < n_way - j; that is, when the + // matrix being partitioned is sufficiently small relative to + // n_way such that there is not even enough work for every + // (remaining) thread to get bf (or bf_left) columns. The + // net effect of this safeguard is that some threads may get + // assigned empty ranges (ie: no work), which of course must + // happen in some situations. + if ( width > n_j ) width = n_j; + + return width; + } + } + + // This block computes the width assuming that we are entirely within + // a dense rectangle that precedes the triangular (or trapezoidal) + // part. + { + // First compute the width of the current panel under the + // assumption that the diagonal offset would not intersect. + width = ( dim_t )bli_round( ( double )area_per_thr / ( double )m ); + + // Adjust the width, if necessary. Specifically, we may need + // to allocate the edge case to the first subpartition, if + // requested; otherwise, we just need to ensure that the + // subpartition is a multiple of the blocking factor. + if ( j == 0 && handle_edge_low ) + { + if ( width % bf != bf_left ) width += bf_left - ( width % bf ); + } + else // if interior case + { + // Round up to the next multiple of the blocking factor. + //if ( width % bf != 0 ) width += bf - ( width % bf ); + // Round to the nearest multiple of the blocking factor. + if ( width % bf != 0 ) width = bli_round_to_mult( width, bf ); + } + } + + // We need to recompute width if the panel, according to the width + // as currently computed, would intersect the diagonal. + if ( diagoff_j < width ) + { + dim_t offm_inc, offn_inc; + + // Prune away the unstored region above the diagonal, if it exists. + // Note that the entire region was pruned initially, so we know that + // we don't need to try to prune the right side. (Also, we discard + // the offset deltas since we don't need to actually index into the + // subpartition.) + bli_prune_unstored_region_top_l( diagoff_j, m, n_j, offm_inc ); + //bli_prune_unstored_region_right_l( diagoff_j, m, n_j, offn_inc ); + + // We don't need offm_inc, offn_inc here. These statements should + // prevent compiler warnings. + ( void )offm_inc; + ( void )offn_inc; + + // Prepare to solve a quadratic equation to find the width of the + // current (jth) subpartition given the m dimension, diagonal offset, + // and area. + // NOTE: We know that the +/- in the quadratic formula must be a + + // here because we know that the desired solution (the subpartition + // width) will be smaller than (m + diagoff), not larger. If you + // don't believe me, draw a picture! + const double a = -0.5; + const double b = ( double )m + ( double )diagoff_j + 0.5; + const double c = -0.5 * ( ( double )diagoff_j * + ( ( double )diagoff_j + 1.0 ) + ) - area_per_thr; + const double r = b * b - 4.0 * a * c; + + // If the quadratic solution is not imaginary, round it and use that + // as our width, but make sure it didn't round to zero. Otherwise, + // discard the quadratic solution and leave width, as previously + // computed, unchanged. + if ( r >= 0.0 ) + { + const double x = ( -b + sqrt( r ) ) / ( 2.0 * a ); + + width = ( dim_t )bli_round( x ); + if ( width == 0 ) width = 1; + } + + // Adjust the width, if necessary. + if ( j == 0 && handle_edge_low ) + { + if ( width % bf != bf_left ) width += bf_left - ( width % bf ); + } + else // if interior case + { + // Round up to the next multiple of the blocking factor. + //if ( width % bf != 0 ) width += bf - ( width % bf ); + // Round to the nearest multiple of the blocking factor. + if ( width % bf != 0 ) width = bli_round_to_mult( width, bf ); + } + } + + // Make sure that the width, after being adjusted, does not cause the + // subpartition to exceed n_j. + if ( width > n_j ) width = n_j; + + return width; +} + +siz_t bli_find_area_trap_l( dim_t m, dim_t n, doff_t diagoff ) +{ + dim_t offm_inc = 0; + dim_t offn_inc = 0; + double tri_area; + double area; + + // Prune away any rectangular region above where the diagonal + // intersects the left edge of the subpartition, if it exists. + bli_prune_unstored_region_top_l( diagoff, m, n, offm_inc ); + + // Prune away any rectangular region to the right of where the + // diagonal intersects the bottom edge of the subpartition, if + // it exists. (This shouldn't ever be needed, since the caller + // would presumably have already performed rightward pruning, + // but it's here just in case.) + bli_prune_unstored_region_right_l( diagoff, m, n, offn_inc ); + + ( void )offm_inc; + ( void )offn_inc; + + // Compute the area of the empty triangle so we can subtract it + // from the area of the rectangle that bounds the subpartition. + if ( bli_intersects_diag_n( diagoff, m, n ) ) + { + double tri_dim = ( double )( n - diagoff - 1 ); + tri_area = tri_dim * ( tri_dim + 1.0 ) / 2.0; + } + else + { + // If the diagonal does not intersect the trapezoid, then + // we can compute the area as a simple rectangle. + tri_area = 0.0; + } + + area = ( double )m * ( double )n - tri_area; + + return ( siz_t )area; +} + +siz_t bli_get_range_weighted( void* thr, + doff_t diagoff, + uplo_t uplo, + dim_t m, + dim_t n, + dim_t bf, + bool_t handle_edge_low, + dim_t* j_start_thr, + dim_t* j_end_thr ) +{ + thrinfo_t* thread = ( thrinfo_t* )thr; + + dim_t n_way = thread->n_way; + dim_t my_id = thread->work_id; + + dim_t bf_left = n % bf; + + dim_t j; + + dim_t off_j; + doff_t diagoff_j; + dim_t n_left; + + dim_t width_j; + + dim_t offm_inc, offn_inc; + + double tri_dim, tri_area; + double area_total, area_per_thr; + + siz_t area = 0; + + // In this function, we assume that the caller has already determined + // that (a) the diagonal intersects the submatrix, and (b) the submatrix + // is either lower- or upper-stored. + + if ( bli_is_lower( uplo ) ) + { + // Prune away the unstored region above the diagonal, if it exists, + // and then to the right of where the diagonal intersects the bottom, + // if it exists. (Also, we discard the offset deltas since we don't + // need to actually index into the subpartition.) + bli_prune_unstored_region_top_l( diagoff, m, n, offm_inc ); + bli_prune_unstored_region_right_l( diagoff, m, n, offn_inc ); + + // We don't need offm_inc, offn_inc here. These statements should + // prevent compiler warnings. + ( void )offm_inc; + ( void )offn_inc; + + // Now that pruning has taken place, we know that diagoff >= 0. + + // Compute the total area of the submatrix, accounting for the + // location of the diagonal, and divide it by the number of ways + // of parallelism. + tri_dim = ( double )( n - diagoff - 1 ); + tri_area = tri_dim * ( tri_dim + 1.0 ) / 2.0; + area_total = ( double )m * ( double )n - tri_area; + area_per_thr = area_total / ( double )n_way; + + // Initialize some variables prior to the loop: the offset to the + // current subpartition, the remainder of the n dimension, and + // the diagonal offset of the current subpartition. + off_j = 0; + diagoff_j = diagoff; + n_left = n; + + // Iterate over the subpartition indices corresponding to each + // thread/caucus participating in the n_way parallelism. + for ( j = 0; j < n_way; ++j ) + { + // Compute the width of the jth subpartition, taking the + // current diagonal offset into account, if needed. + width_j = bli_get_range_width_l( diagoff_j, m, n_left, + j, n_way, + bf, bf_left, + area_per_thr, + handle_edge_low ); + + // If the current thread belongs to caucus j, this is his + // subpartition. So we compute the implied index range and + // end our search. + if ( j == my_id ) + { + *j_start_thr = off_j; + *j_end_thr = off_j + width_j; + + area = bli_find_area_trap_l( m, width_j, diagoff_j ); + + break; + } + + // Shift the current subpartition's starting and diagonal offsets, + // as well as the remainder of the n dimension, according to the + // computed width, and then iterate to the next subpartition. + off_j += width_j; + diagoff_j -= width_j; + n_left -= width_j; + } + } + else // if ( bli_is_upper( uplo ) ) + { + // Express the upper-stored case in terms of the lower-stored case. + + // First, we convert the upper-stored trapezoid to an equivalent + // lower-stored trapezoid by rotating it 180 degrees. + bli_rotate180_trapezoid( diagoff, uplo ); + + // Now that the trapezoid is "flipped" in the n dimension, negate + // the bool that encodes whether to handle the edge case at the + // low (or high) end of the index range. + bli_toggle_bool( handle_edge_low ); + + // Compute the appropriate range for the rotated trapezoid. + area = bli_get_range_weighted( thr, diagoff, uplo, m, n, bf, + handle_edge_low, + j_start_thr, j_end_thr ); + + // Reverse the indexing basis for the subpartition ranges so that + // the indices, relative to left-to-right iteration through the + // unrotated upper-stored trapezoid, map to the correct columns + // (relative to the diagonal). This amounts to subtracting the + // range from n. + bli_reverse_index_direction( *j_start_thr, *j_end_thr, n ); + } + + return area; +} + +siz_t bli_get_range_weighted_l2r( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ) +{ + siz_t area; + + // This function assigns area-weighted ranges in the n dimension + // where the total range spans 0 to n-1 with 0 at the left end and + // n-1 at the right end. + + if ( bli_obj_intersects_diag( *a ) && + bli_obj_is_upper_or_lower( *a ) ) + { + doff_t diagoff = bli_obj_diag_offset( *a ); + uplo_t uplo = bli_obj_uplo( *a ); + dim_t m = bli_obj_length( *a ); + dim_t n = bli_obj_width( *a ); + + // Support implicit transposition. + if ( bli_obj_has_trans( *a ) ) + { + bli_reflect_about_diag( diagoff, uplo, m, n ); + } + + area = bli_get_range_weighted( thr, diagoff, uplo, m, n, bf, + FALSE, start, end ); + } + else // if dense or zeros + { + area = bli_get_range_l2r( thr, a, bf, + start, end ); + } + + return area; +} + +siz_t bli_get_range_weighted_r2l( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ) +{ + siz_t area; + + // This function assigns area-weighted ranges in the n dimension + // where the total range spans 0 to n-1 with 0 at the right end and + // n-1 at the left end. + + if ( bli_obj_intersects_diag( *a ) && + bli_obj_is_upper_or_lower( *a ) ) + { + doff_t diagoff = bli_obj_diag_offset( *a ); + uplo_t uplo = bli_obj_uplo( *a ); + dim_t m = bli_obj_length( *a ); + dim_t n = bli_obj_width( *a ); + + // Support implicit transposition. + if ( bli_obj_has_trans( *a ) ) + { + bli_reflect_about_diag( diagoff, uplo, m, n ); + } + + bli_rotate180_trapezoid( diagoff, uplo ); + + area = bli_get_range_weighted( thr, diagoff, uplo, m, n, bf, + TRUE, start, end ); + } + else // if dense or zeros + { + area = bli_get_range_r2l( thr, a, bf, + start, end ); + } + + return area; +} + +siz_t bli_get_range_weighted_t2b( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ) +{ + siz_t area; + + // This function assigns area-weighted ranges in the m dimension + // where the total range spans 0 to m-1 with 0 at the top end and + // m-1 at the bottom end. + + if ( bli_obj_intersects_diag( *a ) && + bli_obj_is_upper_or_lower( *a ) ) + { + doff_t diagoff = bli_obj_diag_offset( *a ); + uplo_t uplo = bli_obj_uplo( *a ); + dim_t m = bli_obj_length( *a ); + dim_t n = bli_obj_width( *a ); + + // Support implicit transposition. + if ( bli_obj_has_trans( *a ) ) + { + bli_reflect_about_diag( diagoff, uplo, m, n ); + } + + bli_reflect_about_diag( diagoff, uplo, m, n ); + + area = bli_get_range_weighted( thr, diagoff, uplo, m, n, bf, + FALSE, start, end ); + } + else // if dense or zeros + { + area = bli_get_range_t2b( thr, a, bf, + start, end ); + } + + return area; +} + +siz_t bli_get_range_weighted_b2t( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ) +{ + siz_t area; + + // This function assigns area-weighted ranges in the m dimension + // where the total range spans 0 to m-1 with 0 at the bottom end and + // m-1 at the top end. + + if ( bli_obj_intersects_diag( *a ) && + bli_obj_is_upper_or_lower( *a ) ) + { + doff_t diagoff = bli_obj_diag_offset( *a ); + uplo_t uplo = bli_obj_uplo( *a ); + dim_t m = bli_obj_length( *a ); + dim_t n = bli_obj_width( *a ); + + // Support implicit transposition. + if ( bli_obj_has_trans( *a ) ) + { + bli_reflect_about_diag( diagoff, uplo, m, n ); + } + + bli_reflect_about_diag( diagoff, uplo, m, n ); + + bli_rotate180_trapezoid( diagoff, uplo ); + + area = bli_get_range_weighted( thr, diagoff, uplo, m, n, bf, + TRUE, start, end ); + } + else // if dense or zeros + { + area = bli_get_range_b2t( thr, a, bf, + start, end ); + } + + return area; } diff --git a/frame/base/bli_threading.h b/frame/base/bli_threading.h index 923e16509b..5fdff80fd0 100644 --- a/frame/base/bli_threading.h +++ b/frame/base/bli_threading.h @@ -87,6 +87,10 @@ typedef struct thread_comm_s thread_comm_t; #include "bli_threading_omp.h" #include "bli_threading_pthreads.h" +void bli_thread_init( void ); +void bli_thread_finalize( void ); +bool_t bli_thread_is_initialized( void ); + // Thread Communicator Interface Definitions void bli_setup_communicator( thread_comm_t* communicator, dim_t n_threads ); void bli_cleanup_communicator( thread_comm_t* communicator ); @@ -97,13 +101,25 @@ void bli_barrier( thread_comm_t* communicator, dim_t thread_id ); struct thrinfo_s { - thread_comm_t* ocomm; //The thread communicator for the other threads sharing the same work at this level - dim_t ocomm_id; //Our thread id within that thread comm - thread_comm_t* icomm; //The thread communicator for the other threads sharing the same work at this level - dim_t icomm_id; //Our thread id within that thread comm + // The thread communicator for the other threads sharing the same work + // at this level. + thread_comm_t* ocomm; + + // Our thread id within the ocomm thread communicator. + dim_t ocomm_id; + + // The thread communicator for the other threads sharing the same work + // at this level. + thread_comm_t* icomm; - dim_t n_way; //Number of distinct used to parallelize the loop - dim_t work_id; //What we're working on + // Our thread id within the icomm thread communicator. + dim_t icomm_id; + + // The number of distinct threads used to parallelize the loop. + dim_t n_way; + + // What we're working on. + dim_t work_id; }; typedef struct thrinfo_s thrinfo_t; @@ -124,8 +140,38 @@ typedef struct thrinfo_s thrinfo_t; #define thread_obarrier( thread ) bli_barrier( thread->ocomm, thread->ocomm_id ) #define thread_ibarrier( thread ) bli_barrier( thread->icomm, thread->icomm_id ) -void bli_get_range( void* thread, dim_t all_start, dim_t all_end, dim_t block_factor, dim_t* start, dim_t* end ); -void bli_get_range_weighted( void* thr, dim_t all_start, dim_t all_end, dim_t block_factor, bool_t forward, dim_t* start, dim_t* end); +void bli_get_range( void* thr, dim_t n, dim_t bf, bool_t handle_edge_low, + dim_t* start, dim_t* end ); +siz_t bli_get_range_l2r( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ); +siz_t bli_get_range_r2l( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ); +siz_t bli_get_range_t2b( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ); +siz_t bli_get_range_b2t( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ); + +dim_t bli_get_range_width_l( doff_t diagoff_j, + dim_t m, + dim_t n_j, + dim_t j, + dim_t n_way, + dim_t bf, + dim_t bf_left, + double area_per_thr, + bool_t handle_edge_low ); +siz_t bli_find_area_trap_l( dim_t m, dim_t n, doff_t diagoff ); +siz_t bli_get_range_weighted( void* thr, + doff_t diagoff, + uplo_t uplo, + dim_t m, + dim_t n, + dim_t bf, + bool_t handle_edge_low, + dim_t* j_start_thr, + dim_t* j_end_thr ); + +siz_t bli_get_range_weighted_l2r( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ); +siz_t bli_get_range_weighted_r2l( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ); +siz_t bli_get_range_weighted_t2b( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ); +siz_t bli_get_range_weighted_b2t( void* thr, obj_t* a, dim_t bf, dim_t* start, dim_t* end ); + thrinfo_t* bli_create_thread_info( thread_comm_t* ocomm, dim_t ocomm_id, thread_comm_t* icomm, dim_t icomm_id, dim_t n_way, dim_t work_id ); diff --git a/frame/base/bli_threading_pthreads.c b/frame/base/bli_threading_pthreads.c index caf8a8c005..0aad78c2aa 100644 --- a/frame/base/bli_threading_pthreads.c +++ b/frame/base/bli_threading_pthreads.c @@ -36,6 +36,8 @@ #ifdef BLIS_ENABLE_PTHREADS +void* thread_decorator_helper( void* data_void ); + typedef struct thread_data { level3_int_t func; @@ -48,10 +50,15 @@ typedef struct thread_data void* thread; } thread_data_t; -void* thread_decorator_helper( thread_data_t* data ) +void* thread_decorator_helper( void* data_void ) { + thread_data_t* data = data_void; + data->func( data->alpha, data->a, data->b, data->beta, data->c, data->cntl, data->thread ); + + return NULL; } + void bli_level3_thread_decorator( dim_t n_threads, level3_int_t func, obj_t* alpha, @@ -65,7 +72,7 @@ void bli_level3_thread_decorator( dim_t n_threads, pthread_t* pthreads = (pthread_t*) bli_malloc(sizeof(pthread_t) * n_threads); //Saying "datas" is kind of like saying "all y'all" thread_data_t* datas = (thread_data_t*) bli_malloc(sizeof(thread_data_t) * n_threads); - pthread_attr_t* attr = (pthread_attr_t*) bli_malloc(sizeof(pthread_attr_t) * n_threads); + //pthread_attr_t* attr = (pthread_attr_t*) bli_malloc(sizeof(pthread_attr_t) * n_threads); for( int i = 0; i < n_threads; i++ ) { diff --git a/frame/cntl/bli_cntl_init.c b/frame/cntl/bli_cntl_init.c index 8c005c3786..54187d9094 100644 --- a/frame/cntl/bli_cntl_init.c +++ b/frame/cntl/bli_cntl_init.c @@ -34,8 +34,13 @@ #include "blis.h" +static bool_t bli_cntl_is_init = FALSE; + void bli_cntl_init( void ) { + // If the API is already initialized, return early. + if ( bli_cntl_is_initialized() ) return; + // Level-1 bli_scalv_cntl_init(); bli_packv_cntl_init(); @@ -61,6 +66,9 @@ void bli_cntl_init( void ) // Level-3 induced bli_ind_cntl_init(); + + // Mark API as initialized. + bli_cntl_is_init = TRUE; } void bli_cntl_finalize( void ) @@ -90,5 +98,13 @@ void bli_cntl_finalize( void ) // Level-3 induced bli_ind_cntl_finalize(); + + // Mark API as uninitialized. + bli_cntl_is_init = FALSE; +} + +bool_t bli_cntl_is_initialized( void ) +{ + return bli_cntl_is_init; } diff --git a/frame/cntl/bli_cntl_init.h b/frame/cntl/bli_cntl_init.h index c1bce09e05..a3fdf6279f 100644 --- a/frame/cntl/bli_cntl_init.h +++ b/frame/cntl/bli_cntl_init.h @@ -32,5 +32,6 @@ */ -void bli_cntl_init( void ); -void bli_cntl_finalize( void ); +void bli_cntl_init( void ); +void bli_cntl_finalize( void ); +bool_t bli_cntl_is_initialized( void ); diff --git a/frame/include/bli_config_macro_defs.h b/frame/include/bli_config_macro_defs.h new file mode 100644 index 0000000000..3f33c7e66f --- /dev/null +++ b/frame/include/bli_config_macro_defs.h @@ -0,0 +1,178 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef BLIS_CONFIG_MACRO_DEFS_H +#define BLIS_CONFIG_MACRO_DEFS_H + + +// -- INTEGER PROPERTIES ------------------------------------------------------- + +// The bit size of the integer type used to track values such as dimensions, +// strides, diagonal offsets. A value of 32 results in BLIS using 32-bit signed +// integers while 64 results in 64-bit integers. Any other value results in use +// of the C99 type "long int". Note that this ONLY affects integers used +// internally within BLIS as well as those exposed in the native BLAS-like BLIS +// interface. +#ifndef BLIS_INT_TYPE_SIZE +#define BLIS_INT_TYPE_SIZE 64 +#endif + + +// -- FLOATING-POINT PROPERTIES ------------------------------------------------ + +// Enable use of built-in C99 "float complex" and "double complex" types and +// associated overloaded operations and functions? Disabling results in +// scomplex and dcomplex being defined in terms of simple structs. +// NOTE: AVOID USING THIS FEATURE. IT IS PROBABLY BROKEN. +#ifdef BLIS_ENABLE_C99_COMPLEX + // No additional definitions needed. +#else + // Default behavior is disabled. +#endif + + +// -- MULTITHREADING ----------------------------------------------------------- + +// Enable multithreading via POSIX threads. +#ifdef BLIS_ENABLE_PTHREADS + // No additional definitions needed. +#else + // Default behavior is disabled. +#endif + +// Enable multithreading via OpenMP. +#ifdef BLIS_ENABLE_OPENMP + // No additional definitions needed. +#else + // Default behavior is disabled. +#endif + + +// -- MEMORY ALLOCATION -------------------------------------------------------- + +// Size of a virtual memory page. This is used to align certain memory +// buffers which are allocated and used internally. +#ifndef BLIS_PAGE_SIZE +#define BLIS_PAGE_SIZE 4096 +#endif + +// Alignment size (in bytes) needed by the instruction set for aligned +// SIMD/vector instructions. +#ifndef BLIS_SIMD_ALIGN_SIZE +#define BLIS_SIMD_ALIGN_SIZE 32 +#endif + +// Alignment size used to align local stack buffers within macro-kernel +// functions. +#define BLIS_STACK_BUF_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE + +// Alignment size used when allocating memory dynamically from the operating +// system (eg: posix_memalign()). To disable heap alignment and just use +// malloc() instead, set this to 1. +#define BLIS_HEAP_ADDR_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE + +// Alignment size used when sizing leading dimensions of dynamically +// allocated memory. +#define BLIS_HEAP_STRIDE_ALIGN_SIZE BLIS_SIMD_ALIGN_SIZE + +// Alignment size used when allocating blocks to the internal memory +// pool (for packing buffers). +#define BLIS_POOL_ADDR_ALIGN_SIZE BLIS_PAGE_SIZE + + +// -- MIXED DATATYPE SUPPORT --------------------------------------------------- + +// Basic (homogeneous) datatype support always enabled. + +// AVOID ENABLING MIXED DATATYPE SUPPORT! IT IS PROBABLY BROKEN. + +// Enable mixed domain operations? +//#define BLIS_ENABLE_MIXED_DOMAIN_SUPPORT + +// Enable extra mixed precision operations? +//#define BLIS_ENABLE_MIXED_PRECISION_SUPPORT + + +// -- MISCELLANEOUS OPTIONS ---------------------------------------------------- + +// Stay initialized after auto-initialization, unless and until the user +// explicitly calls bli_finalize(). +#ifdef BLIS_DISABLE_STAY_AUTO_INITIALIZED + #undef BLIS_ENABLE_STAY_AUTO_INITIALIZED +#else + // Default behavior is enabled. + #undef BLIS_ENABLE_STAY_AUTO_INITIALIZED // In case user explicitly enabled. + #define BLIS_ENABLE_STAY_AUTO_INITIALIZED +#endif + + +// -- BLAS COMPATIBILITY LAYER ------------------------------------------------- + +// Enable the BLAS compatibility layer? +#ifdef BLIS_DISABLE_BLAS2BLIS + #undef BLIS_ENABLE_BLAS2BLIS +#else + // Default behavior is enabled. + #undef BLIS_ENABLE_BLAS2BLIS // In case user explicitly enabled. + #define BLIS_ENABLE_BLAS2BLIS +#endif + +// The bit size of the integer type used to track values such as dimensions and +// leading dimensions (ie: column strides) within the BLAS compatibility layer. +// A value of 32 results in the compatibility layer using 32-bit signed integers +// while 64 results in 64-bit integers. Any other value results in use of the +// C99 type "long int". Note that this ONLY affects integers used within the +// BLAS compatibility layer. +#ifndef BLIS_BLAS2BLIS_INT_TYPE_SIZE +#define BLIS_BLAS2BLIS_INT_TYPE_SIZE 64 +#endif + + +// -- CBLAS COMPATIBILITY LAYER ------------------------------------------------ + +// Enable the CBLAS compatibility layer? +// NOTE: Enabling CBLAS will automatically enable the BLAS compatibility layer +// regardless of whether or not it was explicitly enabled above. Furthermore, +// the CBLAS compatibility layer will use the integer type size definition +// specified above when defining the size of its own integers (regardless of +// whether the BLAS layer was enabled directly or indirectly). +#ifdef BLIS_ENABLE_CBLAS + // No additional definitions needed. +#else + // Default behavior is disabled. +#endif + + +#endif + diff --git a/frame/include/bli_kernel_macro_defs.h b/frame/include/bli_kernel_macro_defs.h index e94b818c24..5622a098cf 100644 --- a/frame/include/bli_kernel_macro_defs.h +++ b/frame/include/bli_kernel_macro_defs.h @@ -1245,41 +1245,6 @@ #endif -// -- Define micro-panel alignment --------------------------------------------- - -// In this section, we consider each datatype-specific alignment sizes for -// micro-panels of A and B. If any definition is undefined, we define it to -// a safe default value (the size of the datatype). - -// Alignment for micro-panels of A -#ifndef BLIS_UPANEL_A_ALIGN_SIZE_S -#define BLIS_UPANEL_A_ALIGN_SIZE_S BLIS_SIZEOF_S -#endif -#ifndef BLIS_UPANEL_A_ALIGN_SIZE_D -#define BLIS_UPANEL_A_ALIGN_SIZE_D BLIS_SIZEOF_D -#endif -#ifndef BLIS_UPANEL_A_ALIGN_SIZE_C -#define BLIS_UPANEL_A_ALIGN_SIZE_C BLIS_SIZEOF_C -#endif -#ifndef BLIS_UPANEL_A_ALIGN_SIZE_Z -#define BLIS_UPANEL_A_ALIGN_SIZE_Z BLIS_SIZEOF_Z -#endif - -// Alignment for micro-panels of B -#ifndef BLIS_UPANEL_B_ALIGN_SIZE_S -#define BLIS_UPANEL_B_ALIGN_SIZE_S BLIS_SIZEOF_S -#endif -#ifndef BLIS_UPANEL_B_ALIGN_SIZE_D -#define BLIS_UPANEL_B_ALIGN_SIZE_D BLIS_SIZEOF_D -#endif -#ifndef BLIS_UPANEL_B_ALIGN_SIZE_C -#define BLIS_UPANEL_B_ALIGN_SIZE_C BLIS_SIZEOF_C -#endif -#ifndef BLIS_UPANEL_B_ALIGN_SIZE_Z -#define BLIS_UPANEL_B_ALIGN_SIZE_Z BLIS_SIZEOF_Z -#endif - - // -- Kernel blocksize checks -------------------------------------------------- // Verify that cache blocksizes are whole multiples of register blocksizes. diff --git a/frame/include/bli_macro_defs.h b/frame/include/bli_macro_defs.h index 4611df4f9c..797c5d6118 100644 --- a/frame/include/bli_macro_defs.h +++ b/frame/include/bli_macro_defs.h @@ -40,12 +40,27 @@ #ifdef __cplusplus // Language is C++; define restrict as nothing. + #ifndef restrict #define restrict + #endif #elif __STDC_VERSION__ >= 199901L // Language is C99 (or later); do nothing since restrict is recognized. #else // Language is pre-C99; define restrict as nothing. + #ifndef restrict #define restrict + #endif +#endif + + +// -- Define typeof() operator if using non-GNU compiler -- + +#ifndef __GNUC__ + #define typeof __typeof__ +#else + #ifndef typeof + #define typeof __typeof__ + #endif #endif @@ -92,6 +107,11 @@ #define MKSTR(s1) #s1 #define STRINGIFY_INT( s ) MKSTR( s ) +// Fortran-77 name-mangling macros. +#define PASTEF770(name) name ## _ +#define PASTEF77(ch1,name) ch1 ## name ## _ +#define PASTEF772(ch1,ch2,name) ch1 ## ch2 ## name ## _ +#define PASTEF773(ch1,ch2,ch3,name) ch1 ## ch2 ## ch3 ## name ## _ // -- Include other groups of macros @@ -100,7 +120,6 @@ #include "bli_gentprot_macro_defs.h" #include "bli_mem_macro_defs.h" -#include "bli_pool_macro_defs.h" #include "bli_obj_macro_defs.h" #include "bli_param_macro_defs.h" #include "bli_complex_macro_defs.h" diff --git a/frame/include/bli_mem_macro_defs.h b/frame/include/bli_mem_macro_defs.h index b603fdc634..51840b7126 100644 --- a/frame/include/bli_mem_macro_defs.h +++ b/frame/include/bli_mem_macro_defs.h @@ -38,17 +38,25 @@ // Mem entry query +#define bli_mem_pblk( mem_p ) \ +\ + ( &((mem_p)->pblk) ) + #define bli_mem_buffer( mem_p ) \ \ - ( (mem_p)->buf ) + ( bli_pblk_buf_align( bli_mem_pblk( mem_p ) ) ) + +#define bli_mem_buf_sys( mem_p ) \ +\ + ( bli_pblk_buf_sys( bli_mem_pblk( mem_p ) ) ) #define bli_mem_buf_type( mem_p ) \ \ - ( (mem_p)->buf_type ) + ( (mem_p)->buf_type ) #define bli_mem_pool( mem_p ) \ \ - ( (mem_p)->pool ) + ( (mem_p)->pool ) #define bli_mem_size( mem_p ) \ \ @@ -65,24 +73,42 @@ // Mem entry modification +#define bli_mem_set_pblk( pblk_p, mem_p ) \ +{ \ + mem_p->pblk = *(pblk_p); \ +} + #define bli_mem_set_buffer( buf0, mem_p ) \ { \ - mem_p->buf = buf0; \ + bli_pblk_set_buf_align( buf0, &(mem_p->pblk) ); \ +} + +#define bli_mem_set_buf_sys( buf0, mem_p ) \ +{ \ + bli_pblk_set_buf_sys( buf0, &(mem_p->pblk) ); \ } #define bli_mem_set_buf_type( buf_type0, mem_p ) \ { \ - mem_p->buf_type = buf_type0; \ + mem_p->buf_type = buf_type0; \ } #define bli_mem_set_pool( pool0, mem_p ) \ { \ - mem_p->pool = pool0; \ + mem_p->pool = pool0; \ } #define bli_mem_set_size( size0, mem_p ) \ { \ - mem_p->size = size0; \ + mem_p->size = size0; \ +} + +#define bli_mem_clear( mem_p ) \ +{ \ + bli_mem_set_buffer( NULL, mem_p ); \ + bli_mem_set_buf_sys( NULL, mem_p ); \ + bli_mem_set_pool( NULL, mem_p ); \ + bli_mem_set_size( 0, mem_p ); \ } diff --git a/frame/include/bli_mem_pool_macro_defs.h b/frame/include/bli_mem_pool_macro_defs.h deleted file mode 100644 index 6b3a6bc811..0000000000 --- a/frame/include/bli_mem_pool_macro_defs.h +++ /dev/null @@ -1,535 +0,0 @@ -/* - - BLIS - An object-based framework for developing high-performance BLAS-like - libraries. - - Copyright (C) 2014, The University of Texas at Austin - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - Neither the name of The University of Texas at Austin nor the names - of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -#ifndef BLIS_POOL_BLOCKS_MACRO_DEFS_H -#define BLIS_POOL_BLOCKS_MACRO_DEFS_H - - -// -- Memory pool block sizing macros ------------------------------------------ - -// In this file, we compute the memory pool block sizes for A, B, and C for -// each floating-point datatype, and then search for and save the maximum. -// The reason we settle on the largest is to prevent a developer from -// implementing a micro-kernel for one datatype (say, single real) and then -// end up in a situation where the memory pool is not large enough because -// the cache blocksize value of the datatype used to size the pool (e.g. -// double) was not set accordingly. - -// First we compute possible scaling factors for each datatype. These -// scaling factors actually take the form of numerator and denominator -// since we want stay in integer arithmetic. The purpose of the scaling -// factors is to increase the amount of space we reserve for the memory -// pool blocks if one of the packed micro-panels has a "leading dimension" -// that is larger than the register blocksize. (In this case, the leading -// dimension of a micro-panel is the packing register blocksize.) - -// Note that when computing the scaling factor, we have to determine which -// of PACKDIM_MR/DEFAULT_MR and PACKDIM_NR/DEFAULT_NR is greater so that -// the pair of values can be used to scale MAXIMUM_MC and MAXIMUM_NC. This -// is needed ONLY because the amount of space allocated for a block of A -// and a panel of B needs to be such that MR and NR can be swapped (ie: A -// is packed with NR and B is packed with MR). This transformation is -// needed for right-side trsm when inducing an algorithm that (a) has -// favorable access patterns for column-stored C and (b) allows the -// macro-kernel to reuse the existing left-side fused gemmtrsm micro-kernels. -// We cross-multiply so that the comparison can stay in integer arithmetic. - - -// -// Find the larger register blocksize for each datatype. -// -#if BLIS_DEFAULT_MR_S > BLIS_DEFAULT_NR_S -#define BLIS_MAX_MNR_S BLIS_DEFAULT_MR_S -#else -#define BLIS_MAX_MNR_S BLIS_DEFAULT_NR_S -#endif -#if BLIS_DEFAULT_MR_D > BLIS_DEFAULT_NR_D -#define BLIS_MAX_MNR_D BLIS_DEFAULT_MR_D -#else -#define BLIS_MAX_MNR_D BLIS_DEFAULT_NR_D -#endif -#if BLIS_DEFAULT_MR_C > BLIS_DEFAULT_NR_C -#define BLIS_MAX_MNR_C BLIS_DEFAULT_MR_C -#else -#define BLIS_MAX_MNR_C BLIS_DEFAULT_NR_C -#endif -#if BLIS_DEFAULT_MR_Z > BLIS_DEFAULT_NR_Z -#define BLIS_MAX_MNR_Z BLIS_DEFAULT_MR_Z -#else -#define BLIS_MAX_MNR_Z BLIS_DEFAULT_NR_Z -#endif - -// -// Define local maximum cache blocksizes -// - -// NOTE: We define these values here just to more concisely capture the -// increasing of the kc dimension blocksizes by the maximum register -// blocksize, which we do to make room for the nudging up of kc at -// runtime to be a multiple of MR or NR for triangular operations trmm, -// trmm3, and trsm. Also, we divide the induced values by 2 since they are -// defined in terms of real elements, but used (later, when computing -// pool block sizes) in terms of complex elements. - -#define BLIS_MAXIMUM_ASM_MC_S (BLIS_MAXIMUM_MC_S) -#define BLIS_MAXIMUM_ASM_KC_S ((BLIS_MAXIMUM_KC_S + BLIS_MAX_MNR_S)/2) -#define BLIS_MAXIMUM_ASM_NC_S (BLIS_MAXIMUM_NC_S) - -#define BLIS_MAXIMUM_ASM_MC_D (BLIS_MAXIMUM_MC_D) -#define BLIS_MAXIMUM_ASM_KC_D ((BLIS_MAXIMUM_KC_D + BLIS_MAX_MNR_D)/2) -#define BLIS_MAXIMUM_ASM_NC_D (BLIS_MAXIMUM_NC_D) - -#define BLIS_MAXIMUM_ASM_MC_C (BLIS_MAXIMUM_MC_C) -#define BLIS_MAXIMUM_ASM_KC_C ((BLIS_MAXIMUM_KC_C + BLIS_MAX_MNR_C)/2) -#define BLIS_MAXIMUM_ASM_NC_C (BLIS_MAXIMUM_NC_C) - -#define BLIS_MAXIMUM_ASM_MC_Z (BLIS_MAXIMUM_MC_Z) -#define BLIS_MAXIMUM_ASM_KC_Z ((BLIS_MAXIMUM_KC_Z + BLIS_MAX_MNR_Z)/2) -#define BLIS_MAXIMUM_ASM_NC_Z (BLIS_MAXIMUM_NC_Z) - -#define BLIS_MAXIMUM_IND_MC_C (BLIS_MAXIMUM_MC_S) -#define BLIS_MAXIMUM_IND_KC_C ((BLIS_MAXIMUM_KC_S + BLIS_MAX_MNR_S)/2) -#define BLIS_MAXIMUM_IND_NC_C (BLIS_MAXIMUM_NC_S) - -#define BLIS_MAXIMUM_IND_MC_Z (BLIS_MAXIMUM_MC_D) -#define BLIS_MAXIMUM_IND_KC_Z ((BLIS_MAXIMUM_KC_D + BLIS_MAX_MNR_D)/2) -#define BLIS_MAXIMUM_IND_NC_Z (BLIS_MAXIMUM_NC_D) - - -// -// Compute scaling factors for single real. -// -#if ( BLIS_PACKDIM_MR_S * BLIS_DEFAULT_NR_S ) >= \ - ( BLIS_PACKDIM_NR_S * BLIS_DEFAULT_MR_S ) - #define BLIS_PACKDIM_MAXR_S BLIS_PACKDIM_MR_S - #define BLIS_DEFAULT_MAXR_S BLIS_DEFAULT_MR_S -#else - #define BLIS_PACKDIM_MAXR_S BLIS_PACKDIM_NR_S - #define BLIS_DEFAULT_MAXR_S BLIS_DEFAULT_NR_S -#endif - -// -// Compute scaling factors for double real. -// -#if ( BLIS_PACKDIM_MR_D * BLIS_DEFAULT_NR_D ) >= \ - ( BLIS_PACKDIM_NR_D * BLIS_DEFAULT_MR_D ) - #define BLIS_PACKDIM_MAXR_D BLIS_PACKDIM_MR_D - #define BLIS_DEFAULT_MAXR_D BLIS_DEFAULT_MR_D -#else - #define BLIS_PACKDIM_MAXR_D BLIS_PACKDIM_NR_D - #define BLIS_DEFAULT_MAXR_D BLIS_DEFAULT_NR_D -#endif - -// -// Compute scaling factors for single complex. -// -#if ( BLIS_PACKDIM_MR_C * BLIS_DEFAULT_NR_C ) >= \ - ( BLIS_PACKDIM_NR_C * BLIS_DEFAULT_MR_C ) - #define BLIS_PACKDIM_MAXR_C BLIS_PACKDIM_MR_C - #define BLIS_DEFAULT_MAXR_C BLIS_DEFAULT_MR_C -#else - #define BLIS_PACKDIM_MAXR_C BLIS_PACKDIM_NR_C - #define BLIS_DEFAULT_MAXR_C BLIS_DEFAULT_NR_C -#endif - -// -// Compute scaling factors for double complex. -// -#if ( BLIS_PACKDIM_MR_Z * BLIS_DEFAULT_NR_Z ) >= \ - ( BLIS_PACKDIM_NR_Z * BLIS_DEFAULT_MR_Z ) - #define BLIS_PACKDIM_MAXR_Z BLIS_PACKDIM_MR_Z - #define BLIS_DEFAULT_MAXR_Z BLIS_DEFAULT_MR_Z -#else - #define BLIS_PACKDIM_MAXR_Z BLIS_PACKDIM_NR_Z - #define BLIS_DEFAULT_MAXR_Z BLIS_DEFAULT_NR_Z -#endif - - -// Next, we define the dimensions of the pool blocks for each datatype. - -// -// Compute pool dimensions for single real -// -#define BLIS_POOL_ASM_MC_S ( ( BLIS_MAXIMUM_ASM_MC_S * BLIS_PACKDIM_MAXR_S ) \ - / BLIS_DEFAULT_MAXR_S ) -#define BLIS_POOL_ASM_NC_S ( ( BLIS_MAXIMUM_ASM_NC_S * BLIS_PACKDIM_MAXR_S ) \ - / BLIS_DEFAULT_MAXR_S ) -#define BLIS_POOL_ASM_KC_S ( ( BLIS_MAXIMUM_ASM_KC_S * BLIS_PACKDIM_KR_S ) \ - / BLIS_DEFAULT_KR_S ) - -// -// Compute pool dimensions for double real -// -#define BLIS_POOL_ASM_MC_D ( ( BLIS_MAXIMUM_ASM_MC_D * BLIS_PACKDIM_MAXR_D ) \ - / BLIS_DEFAULT_MAXR_D ) -#define BLIS_POOL_ASM_NC_D ( ( BLIS_MAXIMUM_ASM_NC_D * BLIS_PACKDIM_MAXR_D ) \ - / BLIS_DEFAULT_MAXR_D ) -#define BLIS_POOL_ASM_KC_D ( ( BLIS_MAXIMUM_ASM_KC_D * BLIS_PACKDIM_KR_D ) \ - / BLIS_DEFAULT_KR_D ) - -// -// Compute pool dimensions for single complex (native) -// -#define BLIS_POOL_ASM_MC_C ( ( BLIS_MAXIMUM_ASM_MC_C * BLIS_PACKDIM_MAXR_C ) \ - / BLIS_DEFAULT_MAXR_C ) -#define BLIS_POOL_ASM_NC_C ( ( BLIS_MAXIMUM_ASM_NC_C * BLIS_PACKDIM_MAXR_C ) \ - / BLIS_DEFAULT_MAXR_C ) -#define BLIS_POOL_ASM_KC_C ( ( BLIS_MAXIMUM_ASM_KC_C * BLIS_PACKDIM_KR_C ) \ - / BLIS_DEFAULT_KR_C ) - -// -// Compute pool dimensions for double complex (native) -// -#define BLIS_POOL_ASM_MC_Z ( ( BLIS_MAXIMUM_ASM_MC_Z * BLIS_PACKDIM_MAXR_Z ) \ - / BLIS_DEFAULT_MAXR_Z ) -#define BLIS_POOL_ASM_NC_Z ( ( BLIS_MAXIMUM_ASM_NC_Z * BLIS_PACKDIM_MAXR_Z ) \ - / BLIS_DEFAULT_MAXR_Z ) -#define BLIS_POOL_ASM_KC_Z ( ( BLIS_MAXIMUM_ASM_KC_Z * BLIS_PACKDIM_KR_Z ) \ - / BLIS_DEFAULT_KR_Z ) - -// -// Compute pool dimensions for single complex (induced) -// -#define BLIS_POOL_IND_MC_C ( ( BLIS_MAXIMUM_IND_MC_C * BLIS_PACKDIM_MAXR_S ) \ - / BLIS_DEFAULT_MAXR_S ) -#define BLIS_POOL_IND_NC_C ( ( BLIS_MAXIMUM_IND_NC_C * BLIS_PACKDIM_MAXR_S ) \ - / BLIS_DEFAULT_MAXR_S ) -#define BLIS_POOL_IND_KC_C ( ( BLIS_MAXIMUM_IND_KC_C * BLIS_PACKDIM_KR_S ) \ - / BLIS_DEFAULT_KR_S ) - -// -// Compute pool dimensions for double complex (induced) -// -#define BLIS_POOL_IND_MC_Z ( ( BLIS_MAXIMUM_IND_MC_Z * BLIS_PACKDIM_MAXR_D ) \ - / BLIS_DEFAULT_MAXR_D ) -#define BLIS_POOL_IND_NC_Z ( ( BLIS_MAXIMUM_IND_NC_Z * BLIS_PACKDIM_MAXR_D ) \ - / BLIS_DEFAULT_MAXR_D ) -#define BLIS_POOL_IND_KC_Z ( ( BLIS_MAXIMUM_IND_KC_Z * BLIS_PACKDIM_KR_D ) \ - / BLIS_DEFAULT_KR_D ) - - -// Now, we compute the size of each block/panel of A, B, and C for each -// datatype. - -// NOTE: We assume the worst case of unit register blocksizes, and -// therefore add a full micro-panel alignment value to KC. This can -// result in quite a bit of unused space, but it's better than the -// alternative of being bitten by the absolute black plague that -// would result from overflowing a block within the pool. - -// -// Compute memory pool block sizes for single real. -// - -#define BLIS_MK_BLOCK_SIZE_ASM_S ( BLIS_POOL_ASM_MC_S * \ - ( BLIS_POOL_ASM_KC_S + \ - ( BLIS_UPANEL_A_ALIGN_SIZE_S / \ - BLIS_SIZEOF_S ) \ - ) * \ - BLIS_SIZEOF_S \ - ) -#define BLIS_KN_BLOCK_SIZE_ASM_S ( \ - ( BLIS_POOL_ASM_KC_S + \ - ( BLIS_UPANEL_B_ALIGN_SIZE_S / \ - BLIS_SIZEOF_S ) \ - ) * \ - BLIS_POOL_ASM_NC_S * \ - BLIS_SIZEOF_S \ - ) -#define BLIS_MN_BLOCK_SIZE_ASM_S ( BLIS_POOL_ASM_MC_S * \ - BLIS_POOL_ASM_NC_S * \ - BLIS_SIZEOF_S \ - ) - -// -// Compute memory pool block sizes for double real. -// - -#define BLIS_MK_BLOCK_SIZE_ASM_D ( BLIS_POOL_ASM_MC_D * \ - ( BLIS_POOL_ASM_KC_D + \ - ( BLIS_UPANEL_A_ALIGN_SIZE_D / \ - BLIS_SIZEOF_D ) \ - ) * \ - BLIS_SIZEOF_D \ - ) -#define BLIS_KN_BLOCK_SIZE_ASM_D ( \ - ( BLIS_POOL_ASM_KC_D + \ - ( BLIS_UPANEL_B_ALIGN_SIZE_D / \ - BLIS_SIZEOF_D ) \ - ) * \ - BLIS_POOL_ASM_NC_D * \ - BLIS_SIZEOF_D \ - ) -#define BLIS_MN_BLOCK_SIZE_ASM_D ( BLIS_POOL_ASM_MC_D * \ - BLIS_POOL_ASM_NC_D * \ - BLIS_SIZEOF_D \ - ) - -// -// Compute memory pool block sizes for single complex. -// - -#define BLIS_MK_BLOCK_SIZE_ASM_C ( BLIS_POOL_ASM_MC_C * \ - ( BLIS_POOL_ASM_KC_C + \ - ( BLIS_UPANEL_A_ALIGN_SIZE_C / \ - BLIS_SIZEOF_C ) \ - ) * \ - BLIS_SIZEOF_C \ - ) -#define BLIS_KN_BLOCK_SIZE_ASM_C ( \ - ( BLIS_POOL_ASM_KC_C + \ - ( BLIS_UPANEL_B_ALIGN_SIZE_C / \ - BLIS_SIZEOF_C ) \ - ) * \ - BLIS_POOL_ASM_NC_C * \ - BLIS_SIZEOF_C \ - ) -#define BLIS_MN_BLOCK_SIZE_ASM_C ( BLIS_POOL_ASM_MC_C * \ - BLIS_POOL_ASM_NC_C * \ - BLIS_SIZEOF_C \ - ) - -// -// Compute memory pool block sizes for double complex. -// - -#define BLIS_MK_BLOCK_SIZE_ASM_Z ( BLIS_POOL_ASM_MC_Z * \ - ( BLIS_POOL_ASM_KC_Z + \ - ( BLIS_UPANEL_A_ALIGN_SIZE_Z / \ - BLIS_SIZEOF_Z ) \ - ) * \ - BLIS_SIZEOF_Z \ - ) -#define BLIS_KN_BLOCK_SIZE_ASM_Z ( \ - ( BLIS_POOL_ASM_KC_Z + \ - ( BLIS_UPANEL_B_ALIGN_SIZE_Z / \ - BLIS_SIZEOF_Z ) \ - ) * \ - BLIS_POOL_ASM_NC_Z * \ - BLIS_SIZEOF_Z \ - ) -#define BLIS_MN_BLOCK_SIZE_ASM_Z ( BLIS_POOL_ASM_MC_Z * \ - BLIS_POOL_ASM_NC_Z * \ - BLIS_SIZEOF_Z \ - ) - -// -// Compute memory pool block sizes for single complex (induced). -// - -// NOTE: We scale by 3/2 because 3m1 requires 50% more space than other -// algorithms. - -#define BLIS_MK_BLOCK_SIZE_IND_C ( BLIS_POOL_IND_MC_C * \ - ( BLIS_POOL_IND_KC_C + \ - ( BLIS_UPANEL_A_ALIGN_SIZE_C / \ - BLIS_SIZEOF_C ) \ - ) * \ - ( BLIS_SIZEOF_C * \ - 3 \ - ) / 2 \ - ) -#define BLIS_KN_BLOCK_SIZE_IND_C ( \ - ( BLIS_POOL_IND_KC_C + \ - ( BLIS_UPANEL_B_ALIGN_SIZE_C / \ - BLIS_SIZEOF_C ) \ - ) * \ - BLIS_POOL_IND_NC_C * \ - ( BLIS_SIZEOF_C * \ - 3 \ - ) / 2 \ - ) -#define BLIS_MN_BLOCK_SIZE_IND_C ( BLIS_POOL_IND_MC_C * \ - BLIS_POOL_IND_NC_C * \ - ( BLIS_SIZEOF_C * \ - 3 \ - ) / 2 \ - ) - -// -// Compute memory pool block sizes for double complex (induced). -// - -// NOTE: We scale by 3/2 because 3m1 requires 50% more space than other -// algorithms. - -#define BLIS_MK_BLOCK_SIZE_IND_Z ( BLIS_POOL_IND_MC_Z * \ - ( BLIS_POOL_IND_KC_Z + \ - ( BLIS_UPANEL_A_ALIGN_SIZE_Z / \ - BLIS_SIZEOF_Z ) \ - ) * \ - ( BLIS_SIZEOF_Z * \ - 3 \ - ) / 2 \ - ) -#define BLIS_KN_BLOCK_SIZE_IND_Z ( \ - ( BLIS_POOL_IND_KC_Z + \ - ( BLIS_UPANEL_B_ALIGN_SIZE_Z / \ - BLIS_SIZEOF_Z ) \ - ) * \ - BLIS_POOL_IND_NC_Z * \ - ( BLIS_SIZEOF_Z * \ - 3 \ - ) / 2 \ - ) -#define BLIS_MN_BLOCK_SIZE_IND_Z ( BLIS_POOL_IND_MC_Z * \ - BLIS_POOL_IND_NC_Z * \ - ( BLIS_SIZEOF_Z * \ - 3 \ - ) / 2 \ - ) - - -// -- Maximum block size search ------------------------------------------------ - -// In this section, we find the largest of each block size. - -// -// Find the largest block size for blocks of A. -// -#define BLIS_MK_BLOCK_SIZE BLIS_MK_BLOCK_SIZE_ASM_S -#if BLIS_MK_BLOCK_SIZE_ASM_D > BLIS_MK_BLOCK_SIZE -#undef BLIS_MK_BLOCK_SIZE -#define BLIS_MK_BLOCK_SIZE BLIS_MK_BLOCK_SIZE_ASM_D -#endif -#if BLIS_MK_BLOCK_SIZE_ASM_C > BLIS_MK_BLOCK_SIZE -#undef BLIS_MK_BLOCK_SIZE -#define BLIS_MK_BLOCK_SIZE BLIS_MK_BLOCK_SIZE_ASM_C -#endif -#if BLIS_MK_BLOCK_SIZE_ASM_Z > BLIS_MK_BLOCK_SIZE -#undef BLIS_MK_BLOCK_SIZE -#define BLIS_MK_BLOCK_SIZE BLIS_MK_BLOCK_SIZE_ASM_Z -#endif -#if BLIS_MK_BLOCK_SIZE_IND_C > BLIS_MK_BLOCK_SIZE -#undef BLIS_MK_BLOCK_SIZE -#define BLIS_MK_BLOCK_SIZE BLIS_MK_BLOCK_SIZE_IND_C -#endif -#if BLIS_MK_BLOCK_SIZE_IND_Z > BLIS_MK_BLOCK_SIZE -#undef BLIS_MK_BLOCK_SIZE -#define BLIS_MK_BLOCK_SIZE BLIS_MK_BLOCK_SIZE_IND_Z -#endif - -// -// Find the largest block size for panels of B. -// -#define BLIS_KN_BLOCK_SIZE BLIS_KN_BLOCK_SIZE_ASM_S -#if BLIS_KN_BLOCK_SIZE_ASM_D > BLIS_KN_BLOCK_SIZE -#undef BLIS_KN_BLOCK_SIZE -#define BLIS_KN_BLOCK_SIZE BLIS_KN_BLOCK_SIZE_ASM_D -#endif -#if BLIS_KN_BLOCK_SIZE_ASM_C > BLIS_KN_BLOCK_SIZE -#undef BLIS_KN_BLOCK_SIZE -#define BLIS_KN_BLOCK_SIZE BLIS_KN_BLOCK_SIZE_ASM_C -#endif -#if BLIS_KN_BLOCK_SIZE_ASM_Z > BLIS_KN_BLOCK_SIZE -#undef BLIS_KN_BLOCK_SIZE -#define BLIS_KN_BLOCK_SIZE BLIS_KN_BLOCK_SIZE_ASM_Z -#endif -#if BLIS_KN_BLOCK_SIZE_IND_C > BLIS_KN_BLOCK_SIZE -#undef BLIS_KN_BLOCK_SIZE -#define BLIS_KN_BLOCK_SIZE BLIS_KN_BLOCK_SIZE_IND_C -#endif -#if BLIS_KN_BLOCK_SIZE_IND_Z > BLIS_KN_BLOCK_SIZE -#undef BLIS_KN_BLOCK_SIZE -#define BLIS_KN_BLOCK_SIZE BLIS_KN_BLOCK_SIZE_IND_Z -#endif - -// -// Find the largest block size for panels of C. -// -#define BLIS_MN_BLOCK_SIZE BLIS_MN_BLOCK_SIZE_ASM_S -#if BLIS_MN_BLOCK_SIZE_ASM_D > BLIS_MN_BLOCK_SIZE -#undef BLIS_MN_BLOCK_SIZE -#define BLIS_MN_BLOCK_SIZE BLIS_MN_BLOCK_SIZE_ASM_D -#endif -#if BLIS_MN_BLOCK_SIZE_ASM_C > BLIS_MN_BLOCK_SIZE -#undef BLIS_MN_BLOCK_SIZE -#define BLIS_MN_BLOCK_SIZE BLIS_MN_BLOCK_SIZE_ASM_C -#endif -#if BLIS_MN_BLOCK_SIZE_ASM_Z > BLIS_MN_BLOCK_SIZE -#undef BLIS_MN_BLOCK_SIZE -#define BLIS_MN_BLOCK_SIZE BLIS_MN_BLOCK_SIZE_ASM_Z -#endif -#if BLIS_MN_BLOCK_SIZE_IND_C > BLIS_MN_BLOCK_SIZE -#undef BLIS_MN_BLOCK_SIZE -#define BLIS_MN_BLOCK_SIZE BLIS_MN_BLOCK_SIZE_IND_C -#endif -#if BLIS_MN_BLOCK_SIZE_IND_Z > BLIS_MN_BLOCK_SIZE -#undef BLIS_MN_BLOCK_SIZE -#define BLIS_MN_BLOCK_SIZE BLIS_MN_BLOCK_SIZE_IND_Z -#endif - - -// -- Compute pool sizes ------------------------------------------------------- - - -// Define each pool's total size using the block sizes determined above. -// These values are used in bli_mem.c to size the static memory pool -// arrays. - -// -// Pool for MC x KC blocks of A. -// -#define BLIS_MK_POOL_SIZE ( \ - BLIS_NUM_MC_X_KC_BLOCKS * \ - ( BLIS_MK_BLOCK_SIZE + \ - BLIS_CONTIG_ADDR_ALIGN_SIZE \ - ) + \ - BLIS_MAX_PRELOAD_BYTE_OFFSET \ - ) - -// -// Pool for KC x NC panels of B. -// -#define BLIS_KN_POOL_SIZE ( \ - BLIS_NUM_KC_X_NC_BLOCKS * \ - ( BLIS_KN_BLOCK_SIZE + \ - BLIS_CONTIG_ADDR_ALIGN_SIZE \ - ) + \ - BLIS_MAX_PRELOAD_BYTE_OFFSET \ - ) - -// -// Pool for MC x NC panels of C. -// -#define BLIS_MN_POOL_SIZE ( \ - BLIS_NUM_MC_X_NC_BLOCKS * \ - ( BLIS_MN_BLOCK_SIZE + \ - BLIS_CONTIG_ADDR_ALIGN_SIZE \ - ) + \ - BLIS_MAX_PRELOAD_BYTE_OFFSET \ - ) - - -#endif diff --git a/frame/include/bli_obj_macro_defs.h b/frame/include/bli_obj_macro_defs.h index 1de889ed5d..5f126c987f 100644 --- a/frame/include/bli_obj_macro_defs.h +++ b/frame/include/bli_obj_macro_defs.h @@ -156,8 +156,8 @@ #define bli_obj_is_upper_or_lower( obj ) \ \ - ( ( (obj).info & BLIS_UPLO_BITS ) == BLIS_BITVAL_UPPER || \ - ( (obj).info & BLIS_UPLO_BITS ) == BLIS_BITVAL_LOWER ) + ( bli_obj_is_upper( obj ) || \ + bli_obj_is_lower( obj ) ) #define bli_obj_is_dense( obj ) \ \ @@ -378,12 +378,12 @@ } \ } -#define bli_obj_apply_trans( trans, obj )\ +#define bli_obj_apply_trans( trans, obj ) \ { \ (obj).info = ( (obj).info ^ (trans) ); \ } -#define bli_obj_apply_conj( conjval, obj )\ +#define bli_obj_apply_conj( conjval, obj ) \ { \ (obj).info = ( (obj).info ^ (conjval) ); \ } @@ -395,21 +395,25 @@ \ ((obj).root) +#define bli_obj_root_uplo( obj ) \ +\ + bli_obj_uplo( *bli_obj_root( obj ) ) + #define bli_obj_root_is_general( obj ) \ \ - bli_obj_is_general( *bli_obj_root( obj ) ) \ + bli_obj_is_general( *bli_obj_root( obj ) ) #define bli_obj_root_is_hermitian( obj ) \ \ - bli_obj_is_hermitian( *bli_obj_root( obj ) ) \ + bli_obj_is_hermitian( *bli_obj_root( obj ) ) #define bli_obj_root_is_symmetric( obj ) \ \ - bli_obj_is_symmetric( *bli_obj_root( obj ) ) \ + bli_obj_is_symmetric( *bli_obj_root( obj ) ) #define bli_obj_root_is_triangular( obj ) \ \ - bli_obj_is_triangular( *bli_obj_root( obj ) ) \ + bli_obj_is_triangular( *bli_obj_root( obj ) ) #define bli_obj_root_is_herm_or_symm( obj ) \ \ @@ -418,11 +422,11 @@ #define bli_obj_root_is_upper( obj ) \ \ - bli_obj_is_upper( *bli_obj_root( obj ) ) \ + bli_obj_is_upper( *bli_obj_root( obj ) ) #define bli_obj_root_is_lower( obj ) \ \ - bli_obj_is_lower( *bli_obj_root( obj ) ) \ + bli_obj_is_lower( *bli_obj_root( obj ) ) // Root matrix modification @@ -437,11 +441,15 @@ #define bli_obj_length( obj ) \ \ - ((obj).m) + ( (obj).dim[BLIS_M] ) #define bli_obj_width( obj ) \ \ - ((obj).n) + ( (obj).dim[BLIS_N] ) + +#define bli_obj_dim( mdim, obj ) \ +\ + ( (obj).dim[mdim] ) #define bli_obj_min_dim( obj ) \ \ @@ -575,23 +583,38 @@ bli_obj_width_stored( obj ) // Dimension modification +#define bli_obj_set_length( dim_m, obj ) \ +{ \ + (obj).dim[BLIS_M] = dim_m; \ +} + +#define bli_obj_set_width( dim_n, obj ) \ +{ \ + (obj).dim[BLIS_N] = dim_n; \ +} + +#define bli_obj_set_dim( mdim, dim_val, obj ) \ +{ \ + (obj).dim[mdim] = dim_val; \ +} + #define bli_obj_set_dims( dim_m, dim_n, obj ) \ { \ - (obj).m = dim_m; \ - (obj).n = dim_n; \ + bli_obj_set_length( dim_m, obj ); \ + bli_obj_set_width( dim_n, obj ); \ } #define bli_obj_set_dims_with_trans( trans, dim_m, dim_n, obj ) \ { \ if ( bli_does_notrans( trans ) ) \ { \ - (obj).m = dim_m; \ - (obj).n = dim_n; \ + bli_obj_set_length( dim_m, obj ); \ + bli_obj_set_width( dim_n, obj ); \ } \ else \ { \ - (obj).m = dim_n; \ - (obj).n = dim_m; \ + bli_obj_set_length( dim_n, obj ); \ + bli_obj_set_width( dim_m, obj ); \ } \ } @@ -600,15 +623,15 @@ bli_obj_width_stored( obj ) #define bli_obj_row_stride( obj ) \ \ - ((obj).rs) + ( (obj).rs ) #define bli_obj_col_stride( obj ) \ \ - ((obj).cs) + ( (obj).cs ) #define bli_obj_imag_stride( obj ) \ \ - ((obj).is) + ( (obj).is ) #define bli_obj_row_stride_mag( obj ) \ \ @@ -667,41 +690,60 @@ bli_obj_width_stored( obj ) // Offset query -#define bli_obj_row_offset( obj ) \ +#define bli_obj_row_off( obj ) \ \ - ( (obj).offm ) + ( (obj).off[BLIS_M] ) -#define bli_obj_col_offset( obj ) \ +#define bli_obj_col_off( obj ) \ \ - ( (obj).offn ) + ( (obj).off[BLIS_N] ) + +#define bli_obj_off( mdim, obj ) \ +\ + ( (obj).off[mdim] ) // Offset modification +#define bli_obj_set_off( mdim, offset, obj ) \ +{ \ + (obj).off[mdim] = offset; \ +} + #define bli_obj_set_offs( offset_m, offset_n, obj ) \ { \ - (obj).offm = offset_m; \ - (obj).offn = offset_n; \ + bli_obj_set_off( BLIS_M, offset_m, obj ); \ + bli_obj_set_off( BLIS_N, offset_n, obj ); \ } -#define bli_obj_inc_offs( offset_m, offset_n, obj ) \ +#define bli_obj_inc_off( mdim, offset, obj ) \ { \ - (obj).offm += offset_m; \ - (obj).offn += offset_n; \ + (obj).off[mdim] += offset; \ } -#define bli_obj_dec_offs( offset_m, offset_n, obj ) \ +#define bli_obj_inc_offm( offset, obj ) \ { \ - (obj).offm -= offset_m; \ - (obj).offn -= offset_n; \ + bli_obj_inc_off( BLIS_M, offset, obj ); \ } +#define bli_obj_inc_offn( offset, obj ) \ +{ \ + bli_obj_inc_off( BLIS_N, offset, obj ); \ +} + +#define bli_obj_inc_offs( offset_m, offset_n, obj ) \ +{ \ + bli_obj_inc_off( BLIS_M, offset_m, obj ); \ + bli_obj_inc_off( BLIS_N, offset_n, obj ); \ +} + + // Diagonal offset query #define bli_obj_diag_offset( obj ) \ \ - ((obj).diag_off) + ( (obj).diag_off ) #define bli_obj_diag_offset_after_trans( obj ) \ \ @@ -758,7 +800,7 @@ bli_obj_width_stored( obj ) #define bli_obj_buffer( obj ) \ \ - (obj).buffer + ( (obj).buffer ) // Buffer address modification @@ -772,7 +814,7 @@ bli_obj_width_stored( obj ) #define bli_obj_internal_scalar_buffer( obj ) \ \ - &((obj).scalar) + &( (obj).scalar ) // Bufferless scalar field modification @@ -790,7 +832,7 @@ bli_obj_width_stored( obj ) #define bli_obj_elem_size( obj ) \ \ - (obj).elem_size \ + ( (obj).elem_size ) // Element size modification @@ -847,19 +889,19 @@ bli_obj_width_stored( obj ) #define bli_obj_panel_length( obj ) \ \ - ((obj).m_panel) + ( (obj).m_panel ) #define bli_obj_panel_width( obj ) \ \ - ((obj).n_panel) + ( (obj).n_panel ) #define bli_obj_panel_dim( obj ) \ \ - ((obj).pd) + ( (obj).pd ) #define bli_obj_panel_stride( obj ) \ \ - ((obj).ps) + ( (obj).ps ) // Packed panel info modification @@ -965,15 +1007,19 @@ bli_obj_width_stored( obj ) #define bli_obj_buffer_for_const( dt, obj ) \ \ ( void* )( \ - ( ( char* )( (obj).buffer ) ) + dt * BLIS_CONSTANT_SLOT_SIZE \ + ( ( char* )( bli_obj_buffer( obj ) ) ) + \ + ( dim_t )( dt * BLIS_CONSTANT_SLOT_SIZE ) \ ) #define bli_obj_buffer_at_off( obj ) \ \ ( void* )( \ - ( ( char* )( (obj).buffer ) ) + ( dim_t )(obj).elem_size * \ - ( (obj).offn * (obj).cs + \ - (obj).offm * (obj).rs ) \ + ( ( char* )( bli_obj_buffer ( obj ) ) + \ + ( dim_t )( bli_obj_elem_size( obj ) ) * \ + ( bli_obj_col_off( obj ) * bli_obj_col_stride( obj ) + \ + bli_obj_row_off( obj ) * bli_obj_row_stride( obj ) \ + ) \ + ) \ ) #define bli_obj_buffer_for_1x1( dt, obj ) \ @@ -1011,8 +1057,8 @@ bli_obj_width_stored( obj ) dim_t n = bli_obj_width( obj ); \ inc_t rs = bli_obj_row_stride( obj ); \ inc_t cs = bli_obj_col_stride( obj ); \ - dim_t offm = bli_obj_row_offset( obj ); \ - dim_t offn = bli_obj_col_offset( obj ); \ + dim_t offm = bli_obj_row_off( obj ); \ + dim_t offn = bli_obj_col_off( obj ); \ doff_t diag_off = bli_obj_diag_offset( obj ); \ \ bli_obj_set_dims( n, m, obj ); \ @@ -1043,8 +1089,8 @@ bli_obj_width_stored( obj ) { \ dim_t m = bli_obj_length( obj ); \ dim_t n = bli_obj_width( obj ); \ - dim_t offm = bli_obj_row_offset( obj ); \ - dim_t offn = bli_obj_col_offset( obj ); \ + dim_t offm = bli_obj_row_off( obj ); \ + dim_t offn = bli_obj_col_off( obj ); \ doff_t diag_off = bli_obj_diag_offset( obj ); \ \ bli_obj_set_dims( n, m, obj ); \ diff --git a/frame/include/bli_param_macro_defs.h b/frame/include/bli_param_macro_defs.h index f1924d9209..137fa4184a 100644 --- a/frame/include/bli_param_macro_defs.h +++ b/frame/include/bli_param_macro_defs.h @@ -42,15 +42,15 @@ #define bli_is_aligned_to( p, size ) \ \ - ( ( siz_t )(p) % ( siz_t )(size) == 0 ) + ( ( uintptr_t )(p) % ( uintptr_t )(size) == 0 ) #define bli_is_unaligned_to( p, size ) \ \ - ( ( siz_t )(p) % ( siz_t )(size) != 0 ) + ( ( uintptr_t )(p) % ( uintptr_t )(size) != 0 ) #define bli_offset_from_alignment( p, size ) \ \ - ( ( siz_t )(p) % ( siz_t )(size) ) + ( ( uintptr_t )(p) % ( uintptr_t )(size) ) // datatype @@ -144,7 +144,8 @@ #define bli_is_upper_or_lower( uplo ) \ \ - ( bli_is_upper( uplo ) || bli_is_lower( uplo ) ) + ( bli_is_upper( uplo ) || \ + bli_is_lower( uplo ) ) #define bli_is_dense( uplo ) \ \ @@ -377,13 +378,15 @@ ( bli_abs( rs ) != 1 && \ bli_abs( cs ) != 1 ) -#define bli_is_row_tilted( rs, cs ) \ +#define bli_is_row_tilted( m, n, rs, cs ) \ \ - ( bli_abs( cs ) < bli_abs( rs ) ) + ( bli_abs( cs ) == bli_abs( rs ) ? n < m \ + : bli_abs( cs ) < bli_abs( rs ) ) -#define bli_is_col_tilted( rs, cs ) \ +#define bli_is_col_tilted( m, n, rs, cs ) \ \ - ( bli_abs( rs ) < bli_abs( cs ) ) + ( bli_abs( rs ) == bli_abs( cs ) ? m < n \ + : bli_abs( rs ) < bli_abs( cs ) ) #define bli_has_nonunit_inc1( inc1 ) \ \ @@ -470,6 +473,106 @@ ( bli_is_lower( uplo ) && bli_is_strictly_above_diag_n( diagoff, m, n ) ) ) +// pruning-related + +#define bli_prune_unstored_region_top_l( diagoff, m, n, offm_inc ) \ +{ \ + offm_inc = 0; \ +\ + /* If the diagonal intersects the left side of the matrix, + ignore the area above that intersection. */ \ + if ( diagoff < 0 ) \ + { \ + m = m + diagoff; \ + offm_inc = - diagoff; \ + diagoff = 0; \ + } \ +} + +#define bli_prune_unstored_region_right_l( diagoff, m, n, offn_inc ) \ +{ \ + offn_inc = 0; \ +\ + /* If the diagonal intersects the bottom side of the matrix, + ignore the area to the right of that intersection. */ \ + if ( n > diagoff + m ) \ + { \ + n = diagoff + m; \ + } \ +} + +#define bli_prune_unstored_region_left_u( diagoff, m, n, offn_inc ) \ +{ \ + offn_inc = 0; \ +\ + /* If the diagonal intersects the top side of the matrix, + ignore the area to the left of that intersection. */ \ + if ( diagoff > 0 ) \ + { \ + n = n - diagoff; \ + offn_inc = + diagoff; \ + diagoff = 0; \ + } \ +} + +#define bli_prune_unstored_region_bottom_u( diagoff, m, n, offm_inc ) \ +{ \ + offm_inc = 0; \ +\ + /* If the diagonal intersects the right side of the matrix, + ignore the area below that intersection. */ \ + if ( m > -diagoff + n ) \ + { \ + m = -diagoff + n; \ + } \ +} + + +// thread range-related + +#define bli_rotate180_trapezoid( diagoff, uplo ) \ +{ \ + diagoff = n - diagoff - m; \ + bli_toggle_uplo( uplo ); \ +} + +#define bli_reverse_index_direction( start, end, n ) \ +{ \ + dim_t start2 = n - start; \ + dim_t end2 = n - end; \ + start = end2; \ + end = start2; \ +} + +#define bli_reflect_about_diag( diagoff, uplo, m, n ) \ +{ \ + bli_swap_dims( m, n ); \ + bli_negate_diag_offset( diagoff ); \ + bli_toggle_uplo( uplo ); \ +} + + +// mdim_t-related + +#define bli_is_m_dim( mdim ) \ +\ + ( mdim == BLIS_M ) + +#define bli_is_n_dim( mdim ) \ +\ + ( mdim == BLIS_N ) + +#define bli_dim_toggled( mdim ) \ +\ + ( mdim == BLIS_M ? BLIS_N : BLIS_M ) + +#define bli_toggle_dim( mdim ) \ +{ \ + mdim = bli_dim_toggled( mdim ); \ +} + + + // index-related #define bli_is_edge_f( i1, iter, left ) \ @@ -551,7 +654,26 @@ bli_is_io_packed( schema ) || \ bli_is_rpi_packed( schema ) ) +#define bli_is_nat_packed( schema ) \ +\ + ( ( schema & BLIS_PACK_FORMAT_BITS ) == 0 ) + +#define bli_is_ind_packed( schema ) \ +\ + ( ( schema & BLIS_PACK_FORMAT_BITS ) != 0 ) + +// pointer-related + +// p1 = p0 + (num/dem) +#define bli_ptr_add( p1, p0, num, dem ) \ +{ \ + p1 = ( typeof( p1 ) ) \ + ( ( char* )(p0) + ( ( (num) * sizeof( *(p0) ) \ + ) / (dem) \ + ) \ + ); \ +} // return datatype for char @@ -665,7 +787,7 @@ uplo_eff = uploa; \ diagoff_eff = diagoffa_use; \ \ - if ( bli_is_row_tilted( inca, lda ) ) \ + if ( bli_is_row_tilted( n_elem_max, n_iter_max, inca, lda ) ) \ { \ bli_swap_dims( n_iter_max, n_elem_max ); \ bli_swap_incs( inca, lda ); \ @@ -852,8 +974,8 @@ bli_negate_diag_offset( diagoff_eff ); \ } \ \ - if ( bli_is_row_tilted( incb, ldb ) && \ - bli_is_row_tilted( inca, lda ) ) \ + if ( bli_is_row_tilted( n_elem_max, n_iter_max, incb, ldb ) && \ + bli_is_row_tilted( n_elem_max, n_iter_max, inca, lda ) ) \ { \ bli_swap_dims( n_iter_max, n_elem_max ); \ bli_swap_incs( inca, lda ); \ diff --git a/frame/include/bli_scalar_macro_defs.h b/frame/include/bli_scalar_macro_defs.h index 6669c0ab94..32258e86e3 100644 --- a/frame/include/bli_scalar_macro_defs.h +++ b/frame/include/bli_scalar_macro_defs.h @@ -243,6 +243,22 @@ bli_fmax( bli_fabs( a ), \ bli_fabs( b ) ) +// round + +#define bli_round( val ) \ +\ + ( round( val ) ) + +// round_to_mult + +#define bli_round_to_mult( val, mult ) \ +\ + ( guint_t )( ( ( ( guint_t )val + \ + ( guint_t )mult / 2 \ + ) / mult \ + ) * mult \ + ) + // isnan, isinf #define bli_isinf( a ) isinf( a ) diff --git a/frame/include/bli_type_defs.h b/frame/include/bli_type_defs.h index e5d83bef3b..95b7c6c0e6 100644 --- a/frame/include/bli_type_defs.h +++ b/frame/include/bli_type_defs.h @@ -91,6 +91,11 @@ typedef guint_t objbits_t; // object information bit field // -- Real types -- +// Define the number of floating-point types supported, and the size of the +// largest type. +#define BLIS_NUM_FP_TYPES 4 +#define BLIS_MAX_TYPE_SIZE sizeof(dcomplex) + // There are some places where we need to use sizeof() inside of a C // preprocessor #if conditional, and so here we define the various sizes // for those purposes. @@ -487,26 +492,41 @@ typedef enum // -- BLIS misc. structure types ----------------------------------------------- // -// -- Memory pool type -- +// -- Pool block type -- typedef struct { - void** block_ptrs; - gint_t top_index; - siz_t num_blocks; - siz_t block_size; + void* buf_sys; + void* buf_align; +} pblk_t; + +// -- Pool type -- + +typedef struct +{ + pblk_t* block_ptrs; + dim_t block_ptrs_len; + + dim_t top_index; + dim_t num_blocks; + + siz_t block_size; + siz_t align_size; } pool_t; // -- Memory object type -- typedef struct mem_s { - void* buf; + pblk_t pblk; packbuf_t buf_type; pool_t* pool; siz_t size; } mem_t; +// -- Memory block type -- + + // -- Blocksize object type -- typedef struct blksz_s @@ -571,10 +591,8 @@ typedef struct obj_s // Basic fields struct obj_s* root; - dim_t offm; - dim_t offn; - dim_t m; - dim_t n; + dim_t off[2]; + dim_t dim[2]; doff_t diag_off; objbits_t info; @@ -606,10 +624,10 @@ typedef struct obj_s { \ (b).root = (a).root; \ \ - (b).offm = (a).offm; \ - (b).offn = (a).offn; \ - (b).m = (a).m; \ - (b).n = (a).n; \ + (b).off[0] = (a).off[0]; \ + (b).off[1] = (a).off[1]; \ + (b).dim[0] = (a).dim[0]; \ + (b).dim[1] = (a).dim[1]; \ (b).diag_off = (a).diag_off; \ \ (b).info = (a).info; \ @@ -649,8 +667,8 @@ typedef struct obj_s { \ (b).root = (a).root; \ \ - (b).offm = (a).offm; \ - (b).offn = (a).offn; \ + (b).off[0] = (a).off[0]; \ + (b).off[1] = (a).off[1]; \ /* Avoid copying m since it will be overwritten. */ \ /* Avoid copying n since it will be overwritten. */ \ (b).diag_off = (a).diag_off; \ @@ -707,21 +725,30 @@ typedef enum } subpart_t; +// -- Matrix dimension type -- + +typedef enum +{ + BLIS_M = 0, + BLIS_N = 1 +} mdim_t; + + // -- Machine parameter types -- typedef enum { - BLIS_MACH_EPS = 0, - BLIS_MACH_SFMIN, - BLIS_MACH_BASE, - BLIS_MACH_PREC, - BLIS_MACH_NDIGMANT, - BLIS_MACH_RND, - BLIS_MACH_EMIN, - BLIS_MACH_RMIN, - BLIS_MACH_EMAX, - BLIS_MACH_RMAX, - BLIS_MACH_EPS2 + BLIS_MACH_EPS = 0, + BLIS_MACH_SFMIN, + BLIS_MACH_BASE, + BLIS_MACH_PREC, + BLIS_MACH_NDIGMANT, + BLIS_MACH_RND, + BLIS_MACH_EMIN, + BLIS_MACH_RMIN, + BLIS_MACH_EMAX, + BLIS_MACH_RMAX, + BLIS_MACH_EPS2 } machval_t; #define BLIS_NUM_MACH_PARAMS 11 diff --git a/frame/include/blis.h b/frame/include/blis.h index 59efad5755..057c734bfc 100644 --- a/frame/include/blis.h +++ b/frame/include/blis.h @@ -54,6 +54,7 @@ extern "C" { // KNOW WHAT YOU ARE DOING. #include "bli_config.h" +#include "bli_config_macro_defs.h" // -- System headers -- @@ -98,17 +99,21 @@ extern "C" { // -- BLIS memory pool definitions -- -#include "bli_mem_pool_macro_defs.h" +//#include "bli_mem_pool_macro_defs.h" // -- Base operation prototypes -- #include "bli_init.h" +#include "bli_const.h" #include "bli_malloc.h" #include "bli_obj.h" #include "bli_obj_scalar.h" +#include "bli_ind.h" +#include "bli_pool.h" #include "bli_mem.h" #include "bli_part.h" +#include "bli_prune.h" #include "bli_query.h" #include "bli_blocksize.h" #include "bli_func.h" @@ -120,9 +125,7 @@ extern "C" { #include "bli_machval.h" #include "bli_getopt.h" #include "bli_opid.h" -#include "bli_flops.h" #include "bli_cntl.h" -#include "bli_ind.h" #include "bli_info.h" diff --git a/frame/ind/cntl/bli_gemm3m1_cntl.c b/frame/ind/cntl/bli_gemm3m1_cntl.c index ab9c98e7e2..909aedfd57 100644 --- a/frame/ind/cntl/bli_gemm3m1_cntl.c +++ b/frame/ind/cntl/bli_gemm3m1_cntl.c @@ -138,7 +138,7 @@ void bli_gemm3m1_cntl_init() gemm3m1_packa_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3m1_mr, gemm3m1_kr, FALSE, // do NOT invert diagonal @@ -150,7 +150,7 @@ void bli_gemm3m1_cntl_init() gemm3m1_packb_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3m1_kr, gemm3m1_nr, FALSE, // do NOT invert diagonal diff --git a/frame/ind/cntl/bli_gemm3m2_cntl.c b/frame/ind/cntl/bli_gemm3m2_cntl.c index 4b80bf69ff..7e362473ca 100644 --- a/frame/ind/cntl/bli_gemm3m2_cntl.c +++ b/frame/ind/cntl/bli_gemm3m2_cntl.c @@ -146,7 +146,7 @@ void bli_gemm3m2_cntl_init() gemm3m2_packa_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3m2_mr, gemm3m2_kr, FALSE, // do NOT invert diagonal @@ -158,7 +158,7 @@ void bli_gemm3m2_cntl_init() gemm3m2_packb_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3m2_kr, gemm3m2_nr, FALSE, // do NOT invert diagonal diff --git a/frame/ind/cntl/bli_gemm3m3_cntl.c b/frame/ind/cntl/bli_gemm3m3_cntl.c index f2193ca9fb..e444067fdf 100644 --- a/frame/ind/cntl/bli_gemm3m3_cntl.c +++ b/frame/ind/cntl/bli_gemm3m3_cntl.c @@ -144,7 +144,7 @@ void bli_gemm3m3_cntl_init() gemm3m3_packb_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3m3_kr, gemm3m3_nr, FALSE, // do NOT invert diagonal diff --git a/frame/ind/cntl/bli_gemm3mh_cntl.c b/frame/ind/cntl/bli_gemm3mh_cntl.c index 8e0b28a1d4..8f108fa16a 100644 --- a/frame/ind/cntl/bli_gemm3mh_cntl.c +++ b/frame/ind/cntl/bli_gemm3mh_cntl.c @@ -143,7 +143,7 @@ void bli_gemm3mh_cntl_init() gemm3mh_packa_cntl_ro = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3mh_mr, gemm3mh_kr, FALSE, // do NOT invert diagonal @@ -155,7 +155,7 @@ void bli_gemm3mh_cntl_init() gemm3mh_packb_cntl_ro = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3mh_kr, gemm3mh_nr, FALSE, // do NOT invert diagonal @@ -168,7 +168,7 @@ void bli_gemm3mh_cntl_init() gemm3mh_packa_cntl_io = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3mh_mr, gemm3mh_kr, FALSE, // do NOT invert diagonal @@ -180,7 +180,7 @@ void bli_gemm3mh_cntl_init() gemm3mh_packb_cntl_io = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3mh_kr, gemm3mh_nr, FALSE, // do NOT invert diagonal @@ -193,7 +193,7 @@ void bli_gemm3mh_cntl_init() gemm3mh_packa_cntl_rpi = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3mh_mr, gemm3mh_kr, FALSE, // do NOT invert diagonal @@ -205,7 +205,7 @@ void bli_gemm3mh_cntl_init() gemm3mh_packb_cntl_rpi = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3mh_kr, gemm3mh_nr, FALSE, // do NOT invert diagonal diff --git a/frame/ind/cntl/bli_gemm4m1_cntl.c b/frame/ind/cntl/bli_gemm4m1_cntl.c index aa2bfa303f..3fb517d529 100644 --- a/frame/ind/cntl/bli_gemm4m1_cntl.c +++ b/frame/ind/cntl/bli_gemm4m1_cntl.c @@ -135,7 +135,7 @@ void bli_gemm4m1_cntl_init() gemm4m1_packa_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm4m1_mr, gemm4m1_kr, FALSE, // do NOT invert diagonal @@ -147,7 +147,7 @@ void bli_gemm4m1_cntl_init() gemm4m1_packb_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm4m1_kr, gemm4m1_nr, FALSE, // do NOT invert diagonal diff --git a/frame/ind/cntl/bli_gemm4mb_cntl.c b/frame/ind/cntl/bli_gemm4mb_cntl.c index 4dddc69e99..c59ed3a989 100644 --- a/frame/ind/cntl/bli_gemm4mb_cntl.c +++ b/frame/ind/cntl/bli_gemm4mb_cntl.c @@ -137,7 +137,7 @@ void bli_gemm4mb_cntl_init() gemm4mb_packa_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm4mb_mr, gemm4mb_kr, FALSE, // do NOT invert diagonal @@ -149,7 +149,7 @@ void bli_gemm4mb_cntl_init() gemm4mb_packb_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm4mb_kr, gemm4mb_nr, FALSE, // do NOT invert diagonal diff --git a/frame/ind/cntl/bli_gemm4mh_cntl.c b/frame/ind/cntl/bli_gemm4mh_cntl.c index 2ce643fca1..2deb4ee095 100644 --- a/frame/ind/cntl/bli_gemm4mh_cntl.c +++ b/frame/ind/cntl/bli_gemm4mh_cntl.c @@ -145,7 +145,7 @@ void bli_gemm4mh_cntl_init() gemm4mh_packa_cntl_ro = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm4mh_mr, gemm4mh_kr, FALSE, // do NOT invert diagonal @@ -157,7 +157,7 @@ void bli_gemm4mh_cntl_init() gemm4mh_packb_cntl_ro = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm4mh_kr, gemm4mh_nr, FALSE, // do NOT invert diagonal @@ -170,7 +170,7 @@ void bli_gemm4mh_cntl_init() gemm4mh_packa_cntl_io = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm4mh_mr, gemm4mh_kr, FALSE, // do NOT invert diagonal @@ -182,7 +182,7 @@ void bli_gemm4mh_cntl_init() gemm4mh_packb_cntl_io = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm4mh_kr, gemm4mh_nr, FALSE, // do NOT invert diagonal diff --git a/frame/ind/cntl/bli_trsm3m1_cntl.c b/frame/ind/cntl/bli_trsm3m1_cntl.c index 642d9317f4..5c88cd688e 100644 --- a/frame/ind/cntl/bli_trsm3m1_cntl.c +++ b/frame/ind/cntl/bli_trsm3m1_cntl.c @@ -112,7 +112,7 @@ void bli_trsm3m1_cntl_init() trsm3m1_l_packa_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, // IMPORTANT: n dim multiple must be mr to // support right and bottom-right edge cases gemm3m1_mr, @@ -126,7 +126,7 @@ void bli_trsm3m1_cntl_init() trsm3m1_l_packb_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, // IMPORTANT: m dim multiple must be mr since // B_pack is updated (ie: serves as C) in trsm gemm3m1_mr, @@ -141,7 +141,7 @@ void bli_trsm3m1_cntl_init() trsm3m1_r_packa_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3m1_nr, gemm3m1_mr, FALSE, // do NOT invert diagonal @@ -153,7 +153,7 @@ void bli_trsm3m1_cntl_init() trsm3m1_r_packb_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm3m1_mr, gemm3m1_mr, TRUE, // invert diagonal diff --git a/frame/ind/cntl/bli_trsm4m1_cntl.c b/frame/ind/cntl/bli_trsm4m1_cntl.c index 191e46f610..54883c42f2 100644 --- a/frame/ind/cntl/bli_trsm4m1_cntl.c +++ b/frame/ind/cntl/bli_trsm4m1_cntl.c @@ -112,7 +112,7 @@ void bli_trsm4m1_cntl_init() trsm4m1_l_packa_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, // IMPORTANT: n dim multiple must be mr to // support right and bottom-right edge cases gemm4m1_mr, @@ -126,7 +126,7 @@ void bli_trsm4m1_cntl_init() trsm4m1_l_packb_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, // IMPORTANT: m dim multiple must be mr since // B_pack is updated (ie: serves as C) in trsm gemm4m1_mr, @@ -141,7 +141,7 @@ void bli_trsm4m1_cntl_init() trsm4m1_r_packa_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm4m1_nr, gemm4m1_mr, FALSE, // do NOT invert diagonal @@ -153,7 +153,7 @@ void bli_trsm4m1_cntl_init() trsm4m1_r_packb_cntl = bli_packm_cntl_obj_create( BLIS_BLOCKED, - BLIS_VARIANT2, + BLIS_VARIANT1, gemm4m1_mr, gemm4m1_mr, TRUE, // invert diagonal diff --git a/frame/ind/query/bli_bsv_query.c b/frame/ind/query/bli_bsv_query.c index 4c43cb8421..73f4a7738f 100644 --- a/frame/ind/query/bli_bsv_query.c +++ b/frame/ind/query/bli_bsv_query.c @@ -167,10 +167,11 @@ blksz_t* bli_bsv_get_avail_blksz( bszid_t bsv, opid_t oper, num_t dt ) blksz_t* bli_bsv_get_blksz( bszid_t bsv, ind_t method ) { - // Initialize BLIS, if it isn't already initialized. This is - // needed because we have to ensure that the blksz_t objects - // have been created. - bli_init(); + // Initialize the cntl API, if it isn't already initialized. This is + // needed because we have to ensure that the blksz_t objects have + // been created, otherwise this function could return a NULL (or + // garbage) address. + bli_cntl_init(); return *(bli_bsizes[ method ][ bsv ]); } diff --git a/frame/ind/query/bli_ind_query.c b/frame/ind/query/bli_ind_query.c index 6e1e19c3a0..d07abc7aaf 100644 --- a/frame/ind/query/bli_ind_query.c +++ b/frame/ind/query/bli_ind_query.c @@ -34,7 +34,6 @@ #include "blis.h" - static void* bli_ind_oper_fp[BLIS_NUM_IND_METHODS][BLIS_NUM_LEVEL3_OPS] = { /* gemm hemm herk her2k symm syrk, syr2k trmm3 trmm trsm */ @@ -212,14 +211,33 @@ char* bli_ind_oper_get_avail_impl_string( opid_t oper, num_t dt ) // ----------------------------------------------------------------------------- +static bool_t bli_ind_is_init = FALSE; + void bli_ind_init( void ) { + // If the API is already initialized, return early. + if ( bli_ind_is_initialized() ) return; + #ifdef BLIS_ENABLE_INDUCED_SCOMPLEX bli_ind_enable_dt( BLIS_4M1A, BLIS_SCOMPLEX ); #endif #ifdef BLIS_ENABLE_INDUCED_DCOMPLEX bli_ind_enable_dt( BLIS_4M1A, BLIS_DCOMPLEX ); #endif + + // Mark API as initialized. + bli_ind_is_init = TRUE; +} + +void bli_ind_finalize( void ) +{ + // Mark API as uninitialized. + bli_ind_is_init = FALSE; +} + +bool_t bli_ind_is_initialized( void ) +{ + return bli_ind_is_init; } // ----------------------------------------------------------------------------- diff --git a/frame/ind/query/bli_ind_query.h b/frame/ind/query/bli_ind_query.h index ff917389b0..6b1437c940 100644 --- a/frame/ind/query/bli_ind_query.h +++ b/frame/ind/query/bli_ind_query.h @@ -83,6 +83,8 @@ ind_t bli_ind_oper_find_avail( opid_t oper, num_t dt ); char* bli_ind_oper_get_avail_impl_string( opid_t oper, num_t dt ); void bli_ind_init( void ); +void bli_ind_finalize( void ); +bool_t bli_ind_is_initialized( void ); void bli_ind_enable( ind_t method ); void bli_ind_disable( ind_t method ); diff --git a/frame/ind/query/bli_ukr_query.c b/frame/ind/query/bli_ukr_query.c index d36a9d21ab..61d07d6fc0 100644 --- a/frame/ind/query/bli_ukr_query.c +++ b/frame/ind/query/bli_ukr_query.c @@ -200,12 +200,13 @@ func_t* bli_ukr_get_funcs( l3ukr_t ukr, ind_t method ) { func_t** p = bli_ukrs[ method ][ ukr ]; - // Initialize BLIS, if it isn't already initialized. This is - // needed because we have to ensure that the ukr func_t objects - // have been created (and thus contain valid function pointers). - bli_init(); + // Initialize the cntl API, if it isn't already initialized. This is + // needed because we have to ensure that the ukr func_t objects have + // been created (and thus contain valid function pointers). + bli_cntl_init(); - // Avoid dereferencing NULL pointers. + // Avoid dereferencing NULL pointers. (A NULL pointer indicates that + // there is no kernel for the requested kernel type and method.) if ( p == NULL ) return NULL; else return *p; } @@ -214,12 +215,13 @@ func_t* bli_ukr_get_ref_funcs( l3ukr_t ukr ) { func_t** p = bli_ref_ukrs[ ukr ]; - // Initialize BLIS, if it isn't already initialized. This is - // needed because we have to ensure that the ukr func_t objects - // have been created (and thus contain valid function pointers). - bli_init(); + // Initialize the cntl API, if it isn't already initialized. This is + // needed because we have to ensure that the ukr func_t objects have + // been created (and thus contain valid function pointers). + bli_cntl_init(); - // Avoid dereferencing NULL pointers. + // Avoid dereferencing NULL pointers. (A NULL pointer indicates that + // there is no reference kernel for the requested kernel type.) if ( p == NULL ) return NULL; else return *p; } diff --git a/frame/ind/ukernels/trsm/bli_gemmtrsm4m1_l_ukr_ref.c b/frame/ind/ukernels/trsm/bli_gemmtrsm4m1_l_ukr_ref.c index f73106485e..d4d885c43e 100644 --- a/frame/ind/ukernels/trsm/bli_gemmtrsm4m1_l_ukr_ref.c +++ b/frame/ind/ukernels/trsm/bli_gemmtrsm4m1_l_ukr_ref.c @@ -79,6 +79,17 @@ void PASTEMAC(ch,varname)( \ \ dim_t i, j; \ \ +/* +printf( "gemmtrsm4m1_l_ukr: is_a = %lu is_b = %lu\n", is_a, is_b ); \ +PASTEMAC(chr,fprintm)( stdout, "gemmtrsm4m1_l_ukr: a1011p_r", m, k+m, \ + a10_r, 1, PASTEMAC(chr,packmr), "%4.1f", "" ); \ +PASTEMAC(chr,fprintm)( stdout, "gemmtrsm4m1_l_ukr: a1011p_i", m, k+m, \ + a10_i, 1, PASTEMAC(chr,packmr), "%4.1f", "" ); \ +PASTEMAC(chr,fprintm)( stdout, "gemmtrsm4m1_l_ukr: b0111p_r", k+m, n, \ + b01_r, PASTEMAC(chr,packnr), 1, "%4.1f", "" ); \ +PASTEMAC(chr,fprintm)( stdout, "gemmtrsm4m1_l_ukr: b0111p_i", k+m, n, \ + b01_i, PASTEMAC(chr,packnr), 1, "%4.1f", "" ); \ +*/ \ \ /* Copy the contents of c to a temporary buffer ct. */ \ if ( !PASTEMAC(chr,eq0)( alpha_i ) ) \ @@ -99,7 +110,7 @@ void PASTEMAC(ch,varname)( \ \ \ /* b11.r = alpha.r * b11.r - ( a10.r * b01.r - a10.i * b01.i ); - b11.i = alpha.r * b11.r - ( a10.r * b01.i + a10.i * b01.r ); */ \ + b11.i = alpha.r * b11.i - ( a10.r * b01.i + a10.i * b01.r ); */ \ \ bli_auxinfo_set_next_ab( a10_r, b01_i, *data ); \ \ @@ -144,7 +155,12 @@ void PASTEMAC(ch,varname)( \ one_r, \ b11_r, rs_b, cs_b, \ data ); \ -\ +/* +PASTEMAC(chr,fprintm)( stdout, "gemmtrsm4m1_l_ukr: b0111p_r post-gemm", k+m, n, \ + b01_r, PASTEMAC(chr,packnr), 1, "%4.1f", "" ); \ +PASTEMAC(chr,fprintm)( stdout, "gemmtrsm4m1_l_ukr: b0111p_i post-gemm", k+m, n, \ + b01_i, PASTEMAC(chr,packnr), 1, "%4.1f", "" ); \ +*/ \ \ /* b11 = inv(a11) * b11; c11 = b11; */ \ @@ -152,6 +168,13 @@ void PASTEMAC(ch,varname)( \ b11_r, \ c11, rs_c, cs_c, \ data ); \ +\ +/* +PASTEMAC(chr,fprintm)( stdout, "gemmtrsm4m1_l_ukr: b0111p_r after", k+m, n, \ + b01_r, PASTEMAC(chr,packnr), 1, "%4.1f", "" ); \ +PASTEMAC(chr,fprintm)( stdout, "gemmtrsm4m1_l_ukr: b0111p_i after", k+m, n, \ + b01_i, PASTEMAC(chr,packnr), 1, "%4.1f", "" ); \ +*/ \ } INSERT_GENTFUNCCO_BASIC2( gemmtrsm4m1_l_ukr_ref, GEMM_UKERNEL, TRSM4M1_L_UKERNEL ) diff --git a/frame/ind/ukernels/trsm/bli_trsm4m1_l_ukr_ref.c b/frame/ind/ukernels/trsm/bli_trsm4m1_l_ukr_ref.c index d03ef40a4f..e11faabe6b 100644 --- a/frame/ind/ukernels/trsm/bli_trsm4m1_l_ukr_ref.c +++ b/frame/ind/ukernels/trsm/bli_trsm4m1_l_ukr_ref.c @@ -66,6 +66,16 @@ void PASTEMAC(ch,varname)( \ dim_t iter, i, j, l; \ dim_t n_behind; \ \ +/* +PASTEMAC(chr,fprintm)( stdout, "trsm4m1_l_ukr: a11p_r", m, m, \ + a_r, 1, PASTEMAC(chr,packmr), "%4.1f", "" ); \ +PASTEMAC(chr,fprintm)( stdout, "trsm4m1_l_ukr: a11p_i", m, m, \ + a_i, 1, PASTEMAC(chr,packmr), "%4.1f", "" ); \ +PASTEMAC(chr,fprintm)( stdout, "trsm4m1_l_ukr: b11p_r", m, n, \ + b_r, PASTEMAC(chr,packnr), 1, "%4.1f", "" ); \ +PASTEMAC(chr,fprintm)( stdout, "trsm4m1_l_ukr: b11p_i", m, n, \ + b_i, PASTEMAC(chr,packnr), 1, "%4.1f", "" ); \ +*/ \ \ for ( iter = 0; iter < m; ++iter ) \ { \ @@ -136,6 +146,13 @@ void PASTEMAC(ch,varname)( \ PASTEMAC(chr,copys)( beta11c_i, *beta11_i ); \ } \ } \ +\ +/* +PASTEMAC(chr,fprintm)( stdout, "trsm4m1_l_ukr: b11p_r after", m, n, \ + b_r, PASTEMAC(chr,packnr), 1, "%4.1f", "" ); \ +PASTEMAC(chr,fprintm)( stdout, "trsm4m1_l_ukr: b11p_i after", m, n, \ + b_i, PASTEMAC(chr,packnr), 1, "%4.1f", "" ); \ +*/ \ } INSERT_GENTFUNCCO_BASIC0( trsm4m1_l_ukr_ref ) diff --git a/kernels/armv8a/neon/3/bli_gemm_opt_4x4.c b/kernels/armv8a/neon/3/bli_gemm_opt_4x4.c index ba98d8dd32..2a54fe8259 100644 --- a/kernels/armv8a/neon/3/bli_gemm_opt_4x4.c +++ b/kernels/armv8a/neon/3/bli_gemm_opt_4x4.c @@ -1,4 +1,4 @@ -/* + /* BLIS An object-based framework for developing high-performance BLAS-like @@ -36,12 +36,7 @@ #include "blis.h" /* - o 4x4 Single precision micro-kernel fully functional. - o Runnable on ARMv8, compiled with aarch64 GCC. - o Use it together with the armv8 BLIS configuration. o Tested on Juno board. Around 7.3 GFLOPS @ 1.1 GHz. - - December 2014. */ void bli_sgemm_opt_4x4( dim_t k, @@ -55,15 +50,15 @@ void bli_sgemm_opt_4x4( { void* a_next = bli_auxinfo_next_a( data ); void* b_next = bli_auxinfo_next_b( data ); - + dim_t k_iter = k / 4; - dim_t k_left = k % 4; + dim_t k_left = k % 4; -__asm__ volatile +__asm__ volatile ( " \n\t" " \n\t" -" ldr x0,%[aaddr] \n\t" // Load address of A. +" ldr x0,%[aaddr] \n\t" // Load address of A. " ldr x1,%[baddr] \n\t" // Load address of B. " ldr x2,%[caddr] \n\t" // Load address of C. " \n\t" @@ -433,12 +428,8 @@ __asm__ volatile /* - o 4x4 Double precision micro-kernel NOT fully functional yet. - o Runnable on ARMv8, compiled with aarch64 GCC. - o Use it together with the armv8 BLIS configuration. - o Tested on Juno board. Around 3 GFLOPS @ 1.1 GHz. - - December 2014. + * Tested on Juno board. Around 5.6 GFLOPS, 2 A57 cores @ 1.1 GHz. + * Tested on Juno board. Around 4 GFLOPS, 4 A53 cores @ 850 MHz. */ void bli_dgemm_opt_4x4( dim_t k, @@ -458,8 +449,8 @@ void bli_dgemm_opt_4x4( __asm__ volatile ( -" \n\t" -" ldr x0,%[aaddr] \n\t" // Load address of A +" \n\t" +" ldr x0,%[aaddr] \n\t" // Load address of A " ldr x1,%[baddr] \n\t" // Load address of B " ldr x2,%[caddr] \n\t" // Load address of C " \n\t" @@ -476,9 +467,16 @@ __asm__ volatile " \n\t" " ldr x9,%[cs_c] \n\t" // Load cs_c " lsl x10,x9,#3 \n\t" // cs_c * sizeof(double) -" \n\t" -" ldp q0,q1,[x0],32 \n\t" // Load a into quad -" ldp q4,q5,[x1],32 \n\t" // Load b into quad +" lsl x11,x9,#4 \n\t" // 2 * cs_c * sizeof(double) -- AUX. +" lsl x12,x9,#5 \n\t" // 3 * cs_c * sizeof(double) -- AUX. +" \n\t" +" ldr x13,%[rs_c] \n\t" // Load rs_c. +" lsl x14,x13,#3 \n\t" // rs_c * sizeof(double). +" \n\t" +" prfm pldl1keep,[x2,0] \n\t" // Prefetch c. +" prfm pldl1keep,[x2,x10] \n\t" // Prefetch c. +" prfm pldl1keep,[x2,x11] \n\t" // Prefetch c. +" prfm pldl1keep,[x2,x12] \n\t" // Prefetch c. " \n\t" " movi v12.2d,#0 \n\t" // Vector for accummulating column 0 " movi v13.2d,#0 \n\t" // Vector for accummulating column 0 @@ -489,10 +487,33 @@ __asm__ volatile " movi v18.2d,#0 \n\t" // Vector for accummulating column 3 " movi v19.2d,#0 \n\t" // Vector for accummulating column 3 " \n\t" +" movi v20.2d,#0 \n\t" // Vector for accummulating column 0 +" movi v21.2d,#0 \n\t" // Vector for accummulating column 0 +" movi v22.2d,#0 \n\t" // Vector for accummulating column 1 +" movi v23.2d,#0 \n\t" // Vector for accummulating column 1 +" movi v24.2d,#0 \n\t" // Vector for accummulating column 2 +" movi v25.2d,#0 \n\t" // Vector for accummulating column 2 +" movi v26.2d,#0 \n\t" // Vector for accummulating column 3 +" movi v27.2d,#0 \n\t" // Vector for accummulating column 3 +" \n\t" " ld1r {v31.2d},[x8] \n\t" // Load beta " \n\t" +" \n\t" +" cmp x5,#0 \n\t" // If k_iter == 0, jump to k_left. +" beq .DCONSIDERKLEFT \n\t" +" \n\t" +" ldp q0,q1,[x0],32 \n\t" // Load a +" ldp q4,q5,[x1],32 \n\t" // Load b +" \n\t" +" cmp x5,1 \n\t" // If there is just one k_iter, jump to that one. +" beq .DLASTITER \n\t" // (as loop is do-while-like). +" \n\t" +" \n\t" " DLOOP: \n\t" // Body " \n\t" +" prfm pldl1keep,[x0,#1024] \n\t" // Prefetch. +" prfm pldl1keep,[x1,#1024] \n\t" // Prefetch. +" \n\t" " ldp q6,q7,[x1],32 \n\t" // Load b+4 into quad " \n\t" " fmla v12.2d,v0.2d,v4.d[0] \n\t" // Accummulate @@ -525,9 +546,86 @@ __asm__ volatile " fmla v17.2d,v3.2d,v7.d[0] \n\t" // Accummulate " fmla v19.2d,v3.2d,v7.d[1] \n\t" // Accummulate " \n\t" -" add x4,x4,1 \n\t" // i = i+1 -" cmp x4,x5 \n\t" // Continue -" blt DLOOP \n\t" // if i < N +" prfm pldl1keep,[x0,#64] \n\t" // Prefetch. +" prfm pldl1keep,[x1,#64] \n\t" // Prefetch. +" \n\t" +" sub x5,x5,1 \n\t" // i-=1 +" cmp x5,1 \n\t" // Iterate again if we are not in k_iter == 1. +"bne DLOOP \n\t" +" \n\t" +".DLASTITER: \n\t" +" \n\t" +" ldp q6,q7,[x1],32 \n\t" // Load b+4 into quad +" \n\t" +" fmla v12.2d,v0.2d,v4.d[0] \n\t" // Accummulate +" fmla v14.2d,v0.2d,v4.d[1] \n\t" // Accummulate +" \n\t" +" fmla v13.2d,v1.2d,v4.d[0] \n\t" // Accummulate +" fmla v15.2d,v1.2d,v4.d[1] \n\t" // Accummulate +" \n\t" +" ldp q2,q3,[x0],32 \n\t" // Load a+4 into quad +" \n\t" +" fmla v16.2d,v0.2d,v5.d[0] \n\t" // Accummulate +" fmla v18.2d,v0.2d,v5.d[1] \n\t" // Accummulate +" \n\t" +" fmla v17.2d,v1.2d,v5.d[0] \n\t" // Accummulate +" fmla v19.2d,v1.2d,v5.d[1] \n\t" // Accummulate +" \n\t" +" ld1r {v30.2d},[x7] \n\t" // Load alpha. +" \n\t" +" fmla v12.2d,v2.2d,v6.d[0] \n\t" // Accummulate +" fmla v14.2d,v2.2d,v6.d[1] \n\t" // Accummulate +" \n\t" +" fmla v13.2d,v3.2d,v6.d[0] \n\t" // Accummulate +" fmla v15.2d,v3.2d,v6.d[1] \n\t" // Accummulate +" \n\t" +" fmla v16.2d,v2.2d,v7.d[0] \n\t" // Accummulate +" fmla v18.2d,v2.2d,v7.d[1] \n\t" // Accummulate +" \n\t" +" fmla v17.2d,v3.2d,v7.d[0] \n\t" // Accummulate +" fmla v19.2d,v3.2d,v7.d[1] \n\t" // Accummulate +" \n\t" +" .DCONSIDERKLEFT: \n\t" +" cmp x6,0 \n\t" // If k_left == 0, we are done. +" beq .DPOSTACCUM \n\t" // else, we enter the k_left loop. +" \n\t" +".DLOOPKLEFT: \n\t" +" \n\t" +" prfm pldl1keep,[x0,#1024] \n\t" // Prefetch. +" prfm pldl1keep,[x1,#1024] \n\t" // Prefetch. +" \n\t" +" ldp q0,q1,[x0],32 \n\t" // Load a into quad +" ldp q4,q5,[x1],32 \n\t" // Load b into quad +" \n\t" +//" sub x6,x6,1 \n\t" +" \n\t" +" fmla v12.2d,v0.2d,v4.d[0] \n\t" // Accummulate +" fmla v14.2d,v0.2d,v4.d[1] \n\t" // Accummulate +" \n\t" +" fmla v13.2d,v1.2d,v4.d[0] \n\t" // Accummulate +" fmla v15.2d,v1.2d,v4.d[1] \n\t" // Accummulate +" \n\t" +" fmla v16.2d,v0.2d,v5.d[0] \n\t" // Accummulate +" fmla v18.2d,v0.2d,v5.d[1] \n\t" // Accummulate +" \n\t" +" fmla v17.2d,v1.2d,v5.d[0] \n\t" // Accummulate +" fmla v19.2d,v1.2d,v5.d[1] \n\t" // Accummulate +" \n\t" +//" cmp x6,0 \n\t" // Iterate again. +//" bne .DLOOPKLEFT \n\t" // if i!=0. +" \n\t" +" .DPOSTACCUM: \n\t" +" ld1r {v30.2d},[x7] \n\t" // Load alpha. +" \n\t" +" cmp x13,#1 \n\t" // If rs_c != 1 (column-major) +" bne .DGENSTORED \n\t" +" \n\t" +" .DCOLSTORED: \n\t" // C is column-major. +" fcmp d31,#0.0 \n\t" +" beq .DBETAZEROCOLSTORED \n\t" // Taking care of the beta==0 case. +" \n\t" +" ldr x2,%[caddr] \n\t" // Load address of C. +" \n\t" " \n\t" " ldp q0,q1,[x2] \n\t" // Load c into quad and increment by cs_c " add x2,x2,x10 \n\t" @@ -537,38 +635,140 @@ __asm__ volatile " add x2,x2,x10 \n\t" " ldp q6,q7,[x2] \n\t" // Load c into quad and increment by cs_c " \n\t" -" ld1r {v30.2d},[x7] \n\t" // Load alpha " \n\t" -" fmul v0.2d,v0.2d,v31.d[0] \n\t" // Scale by beta -" fmul v1.2d,v1.2d,v31.d[0] \n\t" // Scale by beta -" fmul v2.2d,v2.2d,v31.d[0] \n\t" // Scale by beta -" fmul v3.2d,v3.2d,v31.d[0] \n\t" // Scale by beta -" fmul v4.2d,v4.2d,v31.d[0] \n\t" // Scale by beta -" fmul v5.2d,v5.2d,v31.d[0] \n\t" // Scale by beta -" fmul v6.2d,v6.2d,v31.d[0] \n\t" // Scale by beta -" fmul v7.2d,v7.2d,v31.d[0] \n\t" // Scale by beta +" fmul v20.2d,v0.2d,v31.d[0] \n\t" // Scale by beta +" fmul v21.2d,v1.2d,v31.d[0] \n\t" // Scale by beta +" fmul v22.2d,v2.2d,v31.d[0] \n\t" // Scale by beta +" fmul v23.2d,v3.2d,v31.d[0] \n\t" // Scale by beta +" fmul v24.2d,v4.2d,v31.d[0] \n\t" // Scale by beta +" fmul v25.2d,v5.2d,v31.d[0] \n\t" // Scale by beta +" fmul v26.2d,v6.2d,v31.d[0] \n\t" // Scale by beta +" fmul v27.2d,v7.2d,v31.d[0] \n\t" // Scale by beta " \n\t" " prfm pldl2keep,[x16] \n\t" " prfm pldl2keep,[x17] \n\t" " \n\t" -" fmla v0.2d,v12.2d,v30.d[0] \n\t" // Scale by alpha -" fmla v1.2d,v13.2d,v30.d[0] \n\t" // Scale by alpha -" fmla v2.2d,v14.2d,v30.d[0] \n\t" // Scale by alpha -" fmla v3.2d,v15.2d,v30.d[0] \n\t" // Scale by alpha -" fmla v4.2d,v16.2d,v30.d[0] \n\t" // Scale by alpha -" fmla v5.2d,v17.2d,v30.d[0] \n\t" // Scale by alpha -" fmla v6.2d,v18.2d,v30.d[0] \n\t" // Scale by alpha -" fmla v7.2d,v19.2d,v30.d[0] \n\t" // Scale by alpha +" .DBETAZEROCOLSTORED: \n\t" // If beta==0, we won't read from C (nor scale). " \n\t" " ldr x2,%[caddr] \n\t" // Load address of C " \n\t" -" stp q0,q1,[x2] \n\t" // Store quad into c and increment by cs_c +" fmla v20.2d,v12.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v21.2d,v13.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v22.2d,v14.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v23.2d,v15.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v24.2d,v16.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v25.2d,v17.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v26.2d,v18.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v27.2d,v19.2d,v30.d[0] \n\t" // Scale by alpha +" \n\t" +" stp q20,q21,[x2] \n\t" // Store quad into c and increment by cs_c " add x2,x2,x10 \n\t" -" stp q2,q3,[x2] \n\t" // Store quad into c+4 and increment by cs_c +" stp q22,q23,[x2] \n\t" // Store quad into c+4 and increment by cs_c " add x2,x2,x10 \n\t" -" stp q4,q5,[x2] \n\t" // Store quad into c+8 and increment by cs_c +" stp q24,q25,[x2] \n\t" // Store quad into c+8 and increment by cs_c " add x2,x2,x10 \n\t" -" stp q6,q7,[x2] \n\t" // Store quad into c+16 and increment by cs_c +" stp q26,q27,[x2] \n\t" // Store quad into c+16 and increment by cs_c +" \n\t" +" b .DEND \n\t" // Done (TODO: this obviously needs to be moved down to remove jump). +" \n\t" +" .DGENSTORED: \n\t" // C is general-stride stored. +" \n\t" +" fcmp d31,#0.0 \n\t" +" beq .DBETAZEROGENSTORED \n\t" +" \n\t" // If beta!=0, then we can read from C. +" \n\t" // TODO: this was done fast. Rearrange to remove so many address reloads. +" ldr x2,%[caddr] \n\t" // Load address of C. +" \n\t" +" ld1 {v0.d}[0],[x2],x14 \n\t" // Load c00 into quad and increment by rs_c. +" ld1 {v0.d}[1],[x2],x14 \n\t" // Load c01 into quad and increment by rs_c. +" ld1 {v1.d}[0],[x2],x14 \n\t" // Load c02 into quad and increment by rs_c. +" ld1 {v1.d}[1],[x2],x14 \n\t" // Load c03 into quad and increment by rs_c. +" \n\t" +" ldr x2,%[caddr] \n\t" // Load address of C. +" add x2,x2,x10 \n\t" // c += cs_c. +" \n\t" +" ld1 {v2.d}[0],[x2],x14 \n\t" // Load c10 into quad and increment by rs_c. +" ld1 {v2.d}[1],[x2],x14 \n\t" // Load c11 into quad and increment by rs_c. +" ld1 {v3.d}[0],[x2],x14 \n\t" // Load c12 into quad and increment by rs_c. +" ld1 {v3.d}[1],[x2],x14 \n\t" // Load c13 into quad and increment by rs_c. +" \n\t" +" ldr x2,%[caddr] \n\t" // Load address of C. +" add x2,x2,x10 \n\t" // c += cs_c. +" add x2,x2,x10 \n\t" // c += cs_c. +" \n\t" +" ld1 {v4.d}[0],[x2],x14 \n\t" // Load c10 into quad and increment by rs_c. +" ld1 {v4.d}[1],[x2],x14 \n\t" // Load c11 into quad and increment by rs_c. +" ld1 {v5.d}[0],[x2],x14 \n\t" // Load c12 into quad and increment by rs_c. +" ld1 {v5.d}[1],[x2],x14 \n\t" // Load c13 into quad and increment by rs_c. +" \n\t" +" ldr x2,%[caddr] \n\t" // Load address of C. +" add x2,x2,x10 \n\t" // c += cs_c. +" add x2,x2,x10 \n\t" // c += cs_c. +" add x2,x2,x10 \n\t" // c += cs_c. +" \n\t" +" ld1 {v6.d}[0],[x2],x14 \n\t" // Load c10 into quad and increment by rs_c. +" ld1 {v6.d}[1],[x2],x14 \n\t" // Load c11 into quad and increment by rs_c. +" ld1 {v7.d}[0],[x2],x14 \n\t" // Load c12 into quad and increment by rs_c. +" ld1 {v7.d}[1],[x2],x14 \n\t" // Load c13 into quad and increment by rs_c. +" \n\t" +" prfm pldl1keep,[x16,0] \n\t" // Prefetch. +" prfm pldl1keep,[x17,0] \n\t" // Prefetch. +" \n\t" +" fmul v20.2d,v0.2d,v31.d[0] \n\t" // Scale by beta +" fmul v21.2d,v1.2d,v31.d[0] \n\t" // Scale by beta +" fmul v22.2d,v2.2d,v31.d[0] \n\t" // Scale by beta +" fmul v23.2d,v3.2d,v31.d[0] \n\t" // Scale by beta +" fmul v24.2d,v4.2d,v31.d[0] \n\t" // Scale by beta +" fmul v25.2d,v5.2d,v31.d[0] \n\t" // Scale by beta +" fmul v26.2d,v6.2d,v31.d[0] \n\t" // Scale by beta +" fmul v27.2d,v7.2d,v31.d[0] \n\t" // Scale by beta +" \n\t" +" .DBETAZEROGENSTORED: \n\t" // If beta==0, we cannot read from C (nor scale). +" \n\t" +" fmla v20.2d,v12.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v21.2d,v13.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v22.2d,v14.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v23.2d,v15.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v24.2d,v16.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v25.2d,v17.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v26.2d,v18.2d,v30.d[0] \n\t" // Scale by alpha +" fmla v27.2d,v19.2d,v30.d[0] \n\t" // Scale by alpha +" \n\t" +" ldr x2,%[caddr] \n\t" // Load address of C. +" \n\t" +" st1 {v20.d}[0],[x2],x14 \n\t" // Store c00 into quad and increment by rs_c. +" st1 {v20.d}[1],[x2],x14 \n\t" // Store c01 into quad and increment by rs_c. +" st1 {v21.d}[0],[x2],x14 \n\t" // Store c02 into quad and increment by rs_c. +" st1 {v21.d}[1],[x2],x14 \n\t" // Store c03 into quad and increment by rs_c. +" \n\t" +" ldr x2,%[caddr] \n\t" // Load address of C. +" add x2,x2,x10 \n\t" // c += cs_c. +" \n\t" +" st1 {v22.d}[0],[x2],x14 \n\t" // Store c10 into quad and increment by rs_c. +" st1 {v22.d}[1],[x2],x14 \n\t" // Store c11 into quad and increment by rs_c. +" st1 {v23.d}[0],[x2],x14 \n\t" // Store c12 into quad and increment by rs_c. +" st1 {v23.d}[1],[x2],x14 \n\t" // Store c13 into quad and increment by rs_c. +" \n\t" +" ldr x2,%[caddr] \n\t" // Load address of C. +" add x2,x2,x10 \n\t" // c += cs_c. +" add x2,x2,x10 \n\t" // c += cs_c. +" \n\t" +" st1 {v24.d}[0],[x2],x14 \n\t" // Store c10 into quad and increment by rs_c. +" st1 {v24.d}[1],[x2],x14 \n\t" // Store c11 into quad and increment by rs_c. +" st1 {v25.d}[0],[x2],x14 \n\t" // Store c12 into quad and increment by rs_c. +" st1 {v25.d}[1],[x2],x14 \n\t" // Store c13 into quad and increment by rs_c. +" \n\t" +" ldr x2,%[caddr] \n\t" // Load address of C. +" add x2,x2,x10 \n\t" // c += cs_c. +" add x2,x2,x10 \n\t" // c += cs_c. +" add x2,x2,x10 \n\t" // c += cs_c. +" \n\t" +" st1 {v26.d}[0],[x2],x14 \n\t" // Store c10 into quad and increment by rs_c. +" st1 {v26.d}[1],[x2],x14 \n\t" // Store c11 into quad and increment by rs_c. +" st1 {v27.d}[0],[x2],x14 \n\t" // Store c12 into quad and increment by rs_c. +" st1 {v27.d}[1],[x2],x14 \n\t" // Store c13 into quad and increment by rs_c. +" \n\t" +" .DEND: \n\t" // Done! " \n\t" :// output operands (none) :// input operands @@ -587,13 +787,15 @@ __asm__ volatile "x0","x1","x2", "x4","x5","x6", "x7","x8","x9", - "x10","x11","x12", + "x10","x11","x12","x13","x14","x16","x17", "v0","v1","v2", "v3","v4","v5", "v6","v7","v8", "v9","v10","v11", "v12","v13","v14", "v15","v16","v17","v18","v19", + "v20","v21","v22","v23", + "v24","v25","v26","v27", "v30","v31" ); diff --git a/kernels/x86_64/avx/3/bli_gemm_asm_d8x4.c b/kernels/x86_64/avx/3/bli_gemm_asm_d8x4.c index c143a8a5a8..dd952a57ae 100644 --- a/kernels/x86_64/avx/3/bli_gemm_asm_d8x4.c +++ b/kernels/x86_64/avx/3/bli_gemm_asm_d8x4.c @@ -1382,12 +1382,12 @@ void bli_dgemm_asm_8x4( " \n\t" // ie: aligned, ldim aligned, and " \n\t" // column-stored " \n\t" - "cmpq $8, %%rsi \n\t" // set ZF if (8*rs_c) == 8. - "sete %%bl \n\t" // bl = ( ZF == 1 ? 1 : 0 ); - "testq $31, %%rcx \n\t" // set ZF if c & 32 is zero. - "setz %%bh \n\t" // bh = ( ZF == 0 ? 1 : 0 ); - "testq $31, %%rdi \n\t" // set ZF if (8*cs_c) & 32 is zero. - "setz %%al \n\t" // al = ( ZF == 0 ? 1 : 0 ); + // "cmpq $8, %%rsi \n\t" // set ZF if (8*rs_c) == 8. + // "sete %%bl \n\t" // bl = ( ZF == 1 ? 1 : 0 ); + // "testq $31, %%rcx \n\t" // set ZF if c & 32 is zero. + // "setz %%bh \n\t" // bh = ( ZF == 0 ? 1 : 0 ); + // "testq $31, %%rdi \n\t" // set ZF if (8*cs_c) & 32 is zero. + // "setz %%al \n\t" // al = ( ZF == 0 ? 1 : 0 ); " \n\t" // and(bl,bh) followed by " \n\t" // and(bh,al) will reveal result " \n\t" @@ -1399,9 +1399,10 @@ void bli_dgemm_asm_8x4( " \n\t" " \n\t" " \n\t" // check if aligned/column-stored - "andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. - "andb %%bh, %%al \n\t" // set ZF if bh & al == 1. - "jne .DCOLSTORED \n\t" // jump to column storage case + // "andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. + // "andb %%bh, %%al \n\t" // set ZF if bh & al == 1. + "cmpq $8, %%rsi \n\t" // set ZF if (8*rs_c) == 8. + "je .DCOLSTORED \n\t" // jump to column storage case " \n\t" " \n\t" " \n\t" @@ -1536,53 +1537,53 @@ void bli_dgemm_asm_8x4( ".DCOLSTORED: \n\t" " \n\t" // update c00:c33 " \n\t" - "vmovapd (%%rcx), %%ymm0 \n\t" // load c00:c30, + "vmovups (%%rcx), %%ymm0 \n\t" // load c00:c30, "vmulpd %%ymm2, %%ymm0, %%ymm0 \n\t" // scale by beta, "vaddpd %%ymm9, %%ymm0, %%ymm0 \n\t" // add the gemm result, - "vmovapd %%ymm0, (%%rcx) \n\t" // and store back to memory. + "vmovups %%ymm0, (%%rcx) \n\t" // and store back to memory. "addq %%rdi, %%rcx \n\t" // c += cs_c; " \n\t" - "vmovapd (%%rcx), %%ymm0 \n\t" // load c01:c31, + "vmovups (%%rcx), %%ymm0 \n\t" // load c01:c31, "vmulpd %%ymm2, %%ymm0, %%ymm0 \n\t" // scale by beta, "vaddpd %%ymm11, %%ymm0, %%ymm0 \n\t" // add the gemm result, - "vmovapd %%ymm0, (%%rcx) \n\t" // and store back to memory. + "vmovups %%ymm0, (%%rcx) \n\t" // and store back to memory. "addq %%rdi, %%rcx \n\t" // c += cs_c; " \n\t" - "vmovapd (%%rcx), %%ymm0 \n\t" // load c02:c32, + "vmovups (%%rcx), %%ymm0 \n\t" // load c02:c32, "vmulpd %%ymm2, %%ymm0, %%ymm0 \n\t" // scale by beta, "vaddpd %%ymm13, %%ymm0, %%ymm0 \n\t" // add the gemm result, - "vmovapd %%ymm0, (%%rcx) \n\t" // and store back to memory. + "vmovups %%ymm0, (%%rcx) \n\t" // and store back to memory. "addq %%rdi, %%rcx \n\t" // c += cs_c; " \n\t" - "vmovapd (%%rcx), %%ymm0 \n\t" // load c03:c33, + "vmovups (%%rcx), %%ymm0 \n\t" // load c03:c33, "vmulpd %%ymm2, %%ymm0, %%ymm0 \n\t" // scale by beta, "vaddpd %%ymm15, %%ymm0, %%ymm0 \n\t" // add the gemm result, - "vmovapd %%ymm0, (%%rcx) \n\t" // and store back to memory. + "vmovups %%ymm0, (%%rcx) \n\t" // and store back to memory. " \n\t" " \n\t" // update c40:c73 " \n\t" - "vmovapd (%%rdx), %%ymm0 \n\t" // load c40:c70, + "vmovups (%%rdx), %%ymm0 \n\t" // load c40:c70, "vmulpd %%ymm2, %%ymm0, %%ymm0 \n\t" // scale by beta, "vaddpd %%ymm8, %%ymm0, %%ymm0 \n\t" // add the gemm result, - "vmovapd %%ymm0, (%%rdx) \n\t" // and store back to memory. + "vmovups %%ymm0, (%%rdx) \n\t" // and store back to memory. "addq %%rdi, %%rdx \n\t" // c += cs_c; " \n\t" - "vmovapd (%%rdx), %%ymm0 \n\t" // load c41:c71, + "vmovups (%%rdx), %%ymm0 \n\t" // load c41:c71, "vmulpd %%ymm2, %%ymm0, %%ymm0 \n\t" // scale by beta, "vaddpd %%ymm10, %%ymm0, %%ymm0 \n\t" // add the gemm result, - "vmovapd %%ymm0, (%%rdx) \n\t" // and store back to memory. + "vmovups %%ymm0, (%%rdx) \n\t" // and store back to memory. "addq %%rdi, %%rdx \n\t" // c += cs_c; " \n\t" - "vmovapd (%%rdx), %%ymm0 \n\t" // load c42:c72, + "vmovups (%%rdx), %%ymm0 \n\t" // load c42:c72, "vmulpd %%ymm2, %%ymm0, %%ymm0 \n\t" // scale by beta, "vaddpd %%ymm12, %%ymm0, %%ymm0 \n\t" // add the gemm result, - "vmovapd %%ymm0, (%%rdx) \n\t" // and store back to memory. + "vmovups %%ymm0, (%%rdx) \n\t" // and store back to memory. "addq %%rdi, %%rdx \n\t" // c += cs_c; " \n\t" - "vmovapd (%%rdx), %%ymm0 \n\t" // load c43:c73, + "vmovups (%%rdx), %%ymm0 \n\t" // load c43:c73, "vmulpd %%ymm2, %%ymm0, %%ymm0 \n\t" // scale by beta, "vaddpd %%ymm14, %%ymm0, %%ymm0 \n\t" // add the gemm result, - "vmovapd %%ymm0, (%%rdx) \n\t" // and store back to memory. + "vmovups %%ymm0, (%%rdx) \n\t" // and store back to memory. " \n\t" " \n\t" "jmp .DDONE \n\t" // jump to end. @@ -1591,10 +1592,11 @@ void bli_dgemm_asm_8x4( " \n\t" " \n\t" ".DBETAZERO: \n\t" - " \n\t" // check if aligned/column-stored - "andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. - "andb %%bh, %%al \n\t" // set ZF if bh & al == 1. - "jne .DCOLSTORBZ \n\t" // jump to column storage case + " \n\t" // check if column-stored + // "andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. + // "andb %%bh, %%al \n\t" // set ZF if bh & al == 1. + "cmpq $8, %%rsi \n\t" // set ZF if (8*rs_c) == 8. + "je .DCOLSTORBZ \n\t" // jump to column storage case " \n\t" " \n\t" " \n\t" @@ -1665,29 +1667,29 @@ void bli_dgemm_asm_8x4( ".DCOLSTORBZ: \n\t" " \n\t" // update c00:c33 " \n\t" - "vmovapd %%ymm9, (%%rcx) \n\t" // store c00:c30 + "vmovups %%ymm9, (%%rcx) \n\t" // store c00:c30 "addq %%rdi, %%rcx \n\t" // c += cs_c; " \n\t" - "vmovapd %%ymm11, (%%rcx) \n\t" // store c01:c31 + "vmovups %%ymm11, (%%rcx) \n\t" // store c01:c31 "addq %%rdi, %%rcx \n\t" // c += cs_c; " \n\t" - "vmovapd %%ymm13, (%%rcx) \n\t" // store c02:c32 + "vmovups %%ymm13, (%%rcx) \n\t" // store c02:c32 "addq %%rdi, %%rcx \n\t" // c += cs_c; " \n\t" - "vmovapd %%ymm15, (%%rcx) \n\t" // store c03:c33 + "vmovups %%ymm15, (%%rcx) \n\t" // store c03:c33 " \n\t" " \n\t" // update c40:c73 " \n\t" - "vmovapd %%ymm8, (%%rdx) \n\t" // store c40:c70 + "vmovups %%ymm8, (%%rdx) \n\t" // store c40:c70 "addq %%rdi, %%rdx \n\t" // c += cs_c; " \n\t" - "vmovapd %%ymm10, (%%rdx) \n\t" // store c41:c71 + "vmovups %%ymm10, (%%rdx) \n\t" // store c41:c71 "addq %%rdi, %%rdx \n\t" // c += cs_c; " \n\t" - "vmovapd %%ymm12, (%%rdx) \n\t" // store c42:c72 + "vmovups %%ymm12, (%%rdx) \n\t" // store c42:c72 "addq %%rdi, %%rdx \n\t" // c += cs_c; " \n\t" - "vmovapd %%ymm14, (%%rdx) \n\t" // store c43:c73 + "vmovups %%ymm14, (%%rdx) \n\t" // store c43:c73 " \n\t" " \n\t" " \n\t" diff --git a/kernels/x86_64/avx2/3/bli_gemm_asm_d12x4.c b/kernels/x86_64/avx2/3/bli_gemm_asm_d12x4.c new file mode 100644 index 0000000000..2f9ec6542f --- /dev/null +++ b/kernels/x86_64/avx2/3/bli_gemm_asm_d12x4.c @@ -0,0 +1,1318 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + + +#define SGEMM_INPUT_GS_BETA_NZ \ + "vmovlps (%%rcx ), %%xmm0, %%xmm0 \n\t" \ + "vmovhps (%%rcx,%%rsi,1), %%xmm0, %%xmm0 \n\t" \ + "vmovlps (%%rcx,%%rsi,2), %%xmm1, %%xmm1 \n\t" \ + "vmovhps (%%rcx,%%r13 ), %%xmm1, %%xmm1 \n\t" \ + "vshufps $0x88, %%xmm1, %%xmm0, %%xmm0 \n\t" \ + "vmovlps (%%rcx,%%rsi,4), %%xmm2, %%xmm2 \n\t" \ + "vmovhps (%%rcx,%%r15 ), %%xmm2, %%xmm2 \n\t" \ + "vmovlps (%%rcx,%%r13,2), %%xmm1, %%xmm1 \n\t" \ + "vmovhps (%%rcx,%%r10 ), %%xmm1, %%xmm1 \n\t" \ + "vshufps $0x88, %%xmm1, %%xmm2, %%xmm2 \n\t" \ + "vperm2f128 $0x20, %%ymm2, %%ymm0, %%ymm0 \n\t" + +#define SGEMM_OUTPUT_GS_BETA_NZ \ + "vextractf128 $1, %%ymm0, %%xmm2 \n\t" \ + "vmovss %%xmm0, (%%rcx ) \n\t" \ + "vpermilps $0x39, %%xmm0, %%xmm1 \n\t" \ + "vmovss %%xmm1, (%%rcx,%%rsi,1) \n\t" \ + "vpermilps $0x39, %%xmm1, %%xmm0 \n\t" \ + "vmovss %%xmm0, (%%rcx,%%rsi,2) \n\t" \ + "vpermilps $0x39, %%xmm0, %%xmm1 \n\t" \ + "vmovss %%xmm1, (%%rcx,%%r13 ) \n\t" \ + "vmovss %%xmm2, (%%rcx,%%rsi,4) \n\t" \ + "vpermilps $0x39, %%xmm2, %%xmm1 \n\t" \ + "vmovss %%xmm1, (%%rcx,%%r15 ) \n\t" \ + "vpermilps $0x39, %%xmm1, %%xmm2 \n\t" \ + "vmovss %%xmm2, (%%rcx,%%r13,2) \n\t" \ + "vpermilps $0x39, %%xmm2, %%xmm1 \n\t" \ + "vmovss %%xmm1, (%%rcx,%%r10 ) \n\t" + +void bli_sgemm_asm_24x4( + dim_t k, + float* restrict alpha, + float* restrict a, + float* restrict b, + float* restrict beta, + float* restrict c, inc_t rs_c, inc_t cs_c, + auxinfo_t* data + ) +{ + //void* a_next = bli_auxinfo_next_a( data ); + //void* b_next = bli_auxinfo_next_b( data ); + + dim_t k_iter = k / 4; + dim_t k_left = k % 4; + + __asm__ volatile + ( + " \n\t" + "vzeroall \n\t" // zero all xmm/ymm registers. + " \n\t" + " \n\t" + "movq %2, %%rax \n\t" // load address of a. + "movq %3, %%rbx \n\t" // load address of b. + //"movq %9, %%r15 \n\t" // load address of b_next. + " \n\t" + " \n\t" // initialize loop by pre-loading + "vmovaps 0 * 32(%%rax), %%ymm0 \n\t" + "vmovaps 1 * 32(%%rax), %%ymm1 \n\t" + "vmovaps 2 * 32(%%rax), %%ymm2 \n\t" + " \n\t" + "movq %6, %%rcx \n\t" // load address of c + "movq %8, %%rdi \n\t" // load cs_c + "leaq (,%%rdi,4), %%rdi \n\t" // cs_c *= sizeof(float) + " \n\t" + "leaq (%%rdi,%%rdi,2), %%r13 \n\t" // r13 = 3*cs_c; + "prefetcht0 7 * 4(%%rcx) \n\t" // prefetch c + 0*cs_c + "prefetcht0 7 * 4(%%rcx,%%rdi) \n\t" // prefetch c + 1*cs_c + "prefetcht0 7 * 4(%%rcx,%%rdi,2) \n\t" // prefetch c + 2*cs_c + "prefetcht0 7 * 4(%%rcx,%%r13) \n\t" // prefetch c + 3*cs_c + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "movq %0, %%rsi \n\t" // i = k_iter; + "testq %%rsi, %%rsi \n\t" // check i via logical AND. + "je .SCONSIDKLEFT \n\t" // if i == 0, jump to code that + " \n\t" // contains the k_left loop. + " \n\t" + " \n\t" + ".SLOOPKITER: \n\t" // MAIN LOOP + " \n\t" + " \n\t" + " \n\t" // iteration 0 + "prefetcht0 16 * 32(%%rax) \n\t" + " \n\t" + "vbroadcastss 0 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm4 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm5 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm6 \n\t" + " \n\t" + "vbroadcastss 1 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm7 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm8 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm9 \n\t" + " \n\t" + "vbroadcastss 2 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm11 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm12 \n\t" + " \n\t" + "vbroadcastss 3 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm13 \n\t" + "vmovaps 3 * 32(%%rax), %%ymm0 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm14 \n\t" + "vmovaps 4 * 32(%%rax), %%ymm1 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm15 \n\t" + "vmovaps 5 * 32(%%rax), %%ymm2 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" // iteration 1 + "vbroadcastss 4 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm4 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm5 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm6 \n\t" + " \n\t" + "vbroadcastss 5 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm7 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm8 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm9 \n\t" + " \n\t" + "vbroadcastss 6 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm11 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm12 \n\t" + " \n\t" + "vbroadcastss 7 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm13 \n\t" + "vmovaps 6 * 32(%%rax), %%ymm0 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm14 \n\t" + "vmovaps 7 * 32(%%rax), %%ymm1 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm15 \n\t" + "vmovaps 8 * 32(%%rax), %%ymm2 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" // iteration 2 + "prefetcht0 22 * 32(%%rax) \n\t" + " \n\t" + "vbroadcastss 8 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm4 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm5 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm6 \n\t" + " \n\t" + "vbroadcastss 9 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm7 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm8 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm9 \n\t" + " \n\t" + "vbroadcastss 10 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm11 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm12 \n\t" + " \n\t" + "vbroadcastss 11 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm13 \n\t" + "vmovaps 9 * 32(%%rax), %%ymm0 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm14 \n\t" + "vmovaps 10 * 32(%%rax), %%ymm1 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm15 \n\t" + "vmovaps 11 * 32(%%rax), %%ymm2 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" // iteration 3 + "vbroadcastss 12 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm4 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm5 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm6 \n\t" + " \n\t" + "vbroadcastss 13 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm7 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm8 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm9 \n\t" + " \n\t" + "vbroadcastss 14 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm11 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm12 \n\t" + " \n\t" + "vbroadcastss 15 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm13 \n\t" + "vmovaps 12 * 32(%%rax), %%ymm0 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm14 \n\t" + "vmovaps 13 * 32(%%rax), %%ymm1 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm15 \n\t" + "vmovaps 14 * 32(%%rax), %%ymm2 \n\t" + " \n\t" + " \n\t" + " \n\t" + "addq $4 * 24 * 4, %%rax \n\t" // a += 4*24 (unroll x mr) + "addq $4 * 4 * 4, %%rbx \n\t" // b += 4*4 (unroll x nr) + " \n\t" + " \n\t" + "decq %%rsi \n\t" // i -= 1; + "jne .SLOOPKITER \n\t" // iterate again if i != 0. + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + ".SCONSIDKLEFT: \n\t" + " \n\t" + "movq %1, %%rsi \n\t" // i = k_left; + "testq %%rsi, %%rsi \n\t" // check i via logical AND. + "je .SPOSTACCUM \n\t" // if i == 0, we're done; jump to end. + " \n\t" // else, we prepare to enter k_left loop. + " \n\t" + " \n\t" + ".SLOOPKLEFT: \n\t" // EDGE LOOP + " \n\t" + "prefetcht0 16 * 32(%%rax) \n\t" + " \n\t" + "vbroadcastss 0 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm4 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm5 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm6 \n\t" + " \n\t" + "vbroadcastss 1 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm7 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm8 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm9 \n\t" + " \n\t" + "vbroadcastss 2 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm11 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm12 \n\t" + " \n\t" + "vbroadcastss 3 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm13 \n\t" + "vmovaps 3 * 32(%%rax), %%ymm0 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm14 \n\t" + "vmovaps 4 * 32(%%rax), %%ymm1 \n\t" + "vfmadd231ps %%ymm2, %%ymm3, %%ymm15 \n\t" + "vmovaps 5 * 32(%%rax), %%ymm2 \n\t" + " \n\t" + " \n\t" + " \n\t" + "addq $1 * 24 * 4, %%rax \n\t" // a += 1*24 (unroll x mr) + "addq $1 * 4 * 4, %%rbx \n\t" // b += 1*4 (unroll x nr) + " \n\t" + " \n\t" + "decq %%rsi \n\t" // i -= 1; + "jne .SLOOPKLEFT \n\t" // iterate again if i != 0. + " \n\t" + " \n\t" + " \n\t" + ".SPOSTACCUM: \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "movq %4, %%rax \n\t" // load address of alpha + "movq %5, %%rbx \n\t" // load address of beta + "vbroadcastss (%%rax), %%ymm0 \n\t" // load alpha and duplicate + "vbroadcastss (%%rbx), %%ymm3 \n\t" // load beta and duplicate + " \n\t" + "vmulps %%ymm0, %%ymm4, %%ymm4 \n\t" // scale by alpha + "vmulps %%ymm0, %%ymm5, %%ymm5 \n\t" + "vmulps %%ymm0, %%ymm6, %%ymm6 \n\t" + "vmulps %%ymm0, %%ymm7, %%ymm7 \n\t" + "vmulps %%ymm0, %%ymm8, %%ymm8 \n\t" + "vmulps %%ymm0, %%ymm9, %%ymm9 \n\t" + "vmulps %%ymm0, %%ymm10, %%ymm10 \n\t" + "vmulps %%ymm0, %%ymm11, %%ymm11 \n\t" + "vmulps %%ymm0, %%ymm12, %%ymm12 \n\t" + "vmulps %%ymm0, %%ymm13, %%ymm13 \n\t" + "vmulps %%ymm0, %%ymm14, %%ymm14 \n\t" + "vmulps %%ymm0, %%ymm15, %%ymm15 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "movq %7, %%rsi \n\t" // load rs_c + "leaq (,%%rsi,4), %%rsi \n\t" // rsi = rs_c * sizeof(float) + " \n\t" + "leaq (%%rcx,%%rsi,8), %%rdx \n\t" // rdx = c + 8*rs_c; + "leaq (%%rdx,%%rsi,8), %%r12 \n\t" // r12 = c + 16*rs_c; + " \n\t" + "leaq (%%rsi,%%rsi,2), %%r13 \n\t" // r13 = 3*rs_c; + "leaq (%%rsi,%%rsi,4), %%r15 \n\t" // r15 = 5*rs_c; + "leaq (%%r13,%%rsi,4), %%r10 \n\t" // r10 = 7*rs_c; + " \n\t" + " \n\t" + " \n\t" + " \n\t" // determine if + " \n\t" // c % 32 == 0, AND + " \n\t" // 8*cs_c % 32 == 0, AND + " \n\t" // rs_c == 1 + " \n\t" // ie: aligned, ldim aligned, and + " \n\t" // column-stored + " \n\t" + "cmpq $4, %%rsi \n\t" // set ZF if (4*rs_c) == 4. + "sete %%bl \n\t" // bl = ( ZF == 1 ? 1 : 0 ); + "testq $31, %%rcx \n\t" // set ZF if c & 32 is zero. + "setz %%bh \n\t" // bh = ( ZF == 0 ? 1 : 0 ); + "testq $31, %%rdi \n\t" // set ZF if (4*cs_c) & 32 is zero. + "setz %%al \n\t" // al = ( ZF == 0 ? 1 : 0 ); + " \n\t" // and(bl,bh) followed by + " \n\t" // and(bh,al) will reveal result + " \n\t" + " \n\t" // now avoid loading C if beta == 0 + " \n\t" + "vxorps %%ymm0, %%ymm0, %%ymm0 \n\t" // set ymm0 to zero. + "vucomiss %%xmm0, %%xmm3 \n\t" // set ZF if beta == 0. + "je .SBETAZERO \n\t" // if ZF = 1, jump to beta == 0 case + " \n\t" + " \n\t" + " \n\t" // check if aligned/column-stored + "andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. + "andb %%bh, %%al \n\t" // set ZF if bh & al == 1. + "jne .SCOLSTORED \n\t" // jump to column storage case + " \n\t" + " \n\t" + " \n\t" + ".SGENSTORED: \n\t" + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm4, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm7, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm10, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm13, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + //"addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "movq %%rdx, %%rcx \n\t" // rcx = c + 8*rs_c + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm5, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm8, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm11, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm14, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + //"addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "movq %%r12, %%rcx \n\t" // rcx = c + 16*rs_c + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm6, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm9, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm12, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm15, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + //"addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "jmp .SDONE \n\t" // jump to end. + " \n\t" + " \n\t" + " \n\t" + ".SCOLSTORED: \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213ps %%ymm4, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213ps %%ymm5, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovaps (%%r12), %%ymm2 \n\t" + "vfmadd213ps %%ymm6, %%ymm3, %%ymm2 \n\t" + "vmovaps %%ymm2, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213ps %%ymm7, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213ps %%ymm8, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovaps (%%r12), %%ymm2 \n\t" + "vfmadd213ps %%ymm9, %%ymm3, %%ymm2 \n\t" + "vmovaps %%ymm2, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213ps %%ymm10, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213ps %%ymm11, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovaps (%%r12), %%ymm2 \n\t" + "vfmadd213ps %%ymm12, %%ymm3, %%ymm2 \n\t" + "vmovaps %%ymm2, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213ps %%ymm13, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213ps %%ymm14, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovaps (%%r12), %%ymm2 \n\t" + "vfmadd213ps %%ymm15, %%ymm3, %%ymm2 \n\t" + "vmovaps %%ymm2, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "jmp .SDONE \n\t" // jump to end. + " \n\t" + " \n\t" + " \n\t" + ".SBETAZERO: \n\t" + " \n\t" // check if aligned/column-stored + "andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. + "andb %%bh, %%al \n\t" // set ZF if bh & al == 1. + "jne .SCOLSTORBZ \n\t" // jump to column storage case + " \n\t" + " \n\t" + " \n\t" + ".SGENSTORBZ: \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm4, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm7, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm10, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm13, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + //"addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "movq %%rdx, %%rcx \n\t" // rcx = c + 8*rs_c + " \n\t" + " \n\t" + "vmovaps %%ymm5, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm8, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm11, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm14, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + //"addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "movq %%r12, %%rcx \n\t" // rcx = c + 16*rs_c + " \n\t" + " \n\t" + "vmovaps %%ymm6, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm9, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm12, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm15, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + //"addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "jmp .SDONE \n\t" // jump to end. + " \n\t" + " \n\t" + " \n\t" + ".SCOLSTORBZ: \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm4, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm5, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovaps %%ymm6, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + "vmovaps %%ymm7, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm8, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovaps %%ymm9, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + "vmovaps %%ymm10, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm11, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovaps %%ymm12, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + "vmovaps %%ymm13, (%%rcx) \n\t" + //"addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm14, (%%rdx) \n\t" + //"addq %%rdi, %%rdx \n\t" + "vmovaps %%ymm15, (%%r12) \n\t" + //"addq %%rdi, %%r12 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + ".SDONE: \n\t" + " \n\t" + + : // output operands (none) + : // input operands + "m" (k_iter), // 0 + "m" (k_left), // 1 + "m" (a), // 2 + "m" (b), // 3 + "m" (alpha), // 4 + "m" (beta), // 5 + "m" (c), // 6 + "m" (rs_c), // 7 + "m" (cs_c)/*, // 8 + "m" (b_next), // 9 + "m" (a_next)*/ // 10 + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm6", "xmm7", + "xmm8", "xmm9", "xmm10", "xmm11", + "xmm12", "xmm13", "xmm14", "xmm15", + "memory" + ); +} + +#define DGEMM_INPUT_GS_BETA_NZ \ + "vmovlpd (%%rcx ), %%xmm0, %%xmm0 \n\t" \ + "vmovhpd (%%rcx,%%rsi,1), %%xmm0, %%xmm0 \n\t" \ + "vmovlpd (%%rcx,%%rsi,2), %%xmm1, %%xmm1 \n\t" \ + "vmovhpd (%%rcx,%%r13 ), %%xmm1, %%xmm1 \n\t" \ + "vperm2f128 $0x20, %%ymm1, %%ymm0, %%ymm0 \n\t" /*\ + "vmovlps (%%rcx,%%rsi,4), %%xmm2, %%xmm2 \n\t" \ + "vmovhps (%%rcx,%%r15 ), %%xmm2, %%xmm2 \n\t" \ + "vmovlps (%%rcx,%%r13,2), %%xmm1, %%xmm1 \n\t" \ + "vmovhps (%%rcx,%%r10 ), %%xmm1, %%xmm1 \n\t" \ + "vperm2f128 $0x20, %%ymm1, %%ymm2, %%ymm2 \n\t"*/ + +#define DGEMM_OUTPUT_GS_BETA_NZ \ + "vextractf128 $1, %%ymm0, %%xmm1 \n\t" \ + "vmovlpd %%xmm0, (%%rcx ) \n\t" \ + "vmovhpd %%xmm0, (%%rcx,%%rsi ) \n\t" \ + "vmovlpd %%xmm1, (%%rcx,%%rsi,2) \n\t" \ + "vmovhpd %%xmm1, (%%rcx,%%r13 ) \n\t" /*\ + "vextractf128 $1, %%ymm2, %%xmm1 \n\t" \ + "vmovlpd %%xmm2, (%%rcx,%%rsi,4) \n\t" \ + "vmovhpd %%xmm2, (%%rcx,%%r15 ) \n\t" \ + "vmovlpd %%xmm1, (%%rcx,%%r13,2) \n\t" \ + "vmovhpd %%xmm1, (%%rcx,%%r10 ) \n\t"*/ + +void bli_dgemm_asm_12x4( + dim_t k, + double* restrict alpha, + double* restrict a, + double* restrict b, + double* restrict beta, + double* restrict c, inc_t rs_c, inc_t cs_c, + auxinfo_t* data + ) +{ + //void* a_next = bli_auxinfo_next_a( data ); + //void* b_next = bli_auxinfo_next_b( data ); + + dim_t k_iter = k / 4; + dim_t k_left = k % 4; + + __asm__ volatile + ( + " \n\t" + "vzeroall \n\t" // zero all xmm/ymm registers. + " \n\t" + " \n\t" + "movq %2, %%rax \n\t" // load address of a. + "movq %3, %%rbx \n\t" // load address of b. + //"movq %9, %%r15 \n\t" // load address of b_next. + " \n\t" + " \n\t" // initialize loop by pre-loading + "vmovapd 0 * 32(%%rax), %%ymm0 \n\t" + "vmovapd 1 * 32(%%rax), %%ymm1 \n\t" + "vmovapd 2 * 32(%%rax), %%ymm2 \n\t" + " \n\t" + "movq %6, %%rcx \n\t" // load address of c + "movq %8, %%rdi \n\t" // load cs_c + "leaq (,%%rdi,8), %%rdi \n\t" // cs_c *= sizeof(double) + " \n\t" + "leaq (%%rdi,%%rdi,2), %%r13 \n\t" // r13 = 3*cs_c; + "prefetcht0 7 * 8(%%rcx) \n\t" // prefetch c + 0*cs_c + "prefetcht0 7 * 8(%%rcx,%%rdi) \n\t" // prefetch c + 1*cs_c + "prefetcht0 7 * 8(%%rcx,%%rdi,2) \n\t" // prefetch c + 2*cs_c + "prefetcht0 7 * 8(%%rcx,%%r13) \n\t" // prefetch c + 3*cs_c + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "movq %0, %%rsi \n\t" // i = k_iter; + "testq %%rsi, %%rsi \n\t" // check i via logical AND. + "je .DCONSIDKLEFT \n\t" // if i == 0, jump to code that + " \n\t" // contains the k_left loop. + " \n\t" + " \n\t" + ".DLOOPKITER: \n\t" // MAIN LOOP + " \n\t" + " \n\t" + " \n\t" // iteration 0 + "prefetcht0 16 * 32(%%rax) \n\t" + " \n\t" + "vbroadcastsd 0 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm4 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm5 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm6 \n\t" + " \n\t" + "vbroadcastsd 1 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm7 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm8 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm9 \n\t" + " \n\t" + "vbroadcastsd 2 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm11 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm12 \n\t" + " \n\t" + "vbroadcastsd 3 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm13 \n\t" + "vmovapd 3 * 32(%%rax), %%ymm0 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm14 \n\t" + "vmovapd 4 * 32(%%rax), %%ymm1 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm15 \n\t" + "vmovapd 5 * 32(%%rax), %%ymm2 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" // iteration 1 + "vbroadcastsd 4 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm4 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm5 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm6 \n\t" + " \n\t" + "vbroadcastsd 5 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm7 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm8 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm9 \n\t" + " \n\t" + "vbroadcastsd 6 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm11 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm12 \n\t" + " \n\t" + "vbroadcastsd 7 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm13 \n\t" + "vmovapd 6 * 32(%%rax), %%ymm0 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm14 \n\t" + "vmovapd 7 * 32(%%rax), %%ymm1 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm15 \n\t" + "vmovapd 8 * 32(%%rax), %%ymm2 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" // iteration 2 + "prefetcht0 22 * 32(%%rax) \n\t" + " \n\t" + "vbroadcastsd 8 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm4 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm5 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm6 \n\t" + " \n\t" + "vbroadcastsd 9 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm7 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm8 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm9 \n\t" + " \n\t" + "vbroadcastsd 10 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm11 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm12 \n\t" + " \n\t" + "vbroadcastsd 11 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm13 \n\t" + "vmovapd 9 * 32(%%rax), %%ymm0 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm14 \n\t" + "vmovapd 10 * 32(%%rax), %%ymm1 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm15 \n\t" + "vmovapd 11 * 32(%%rax), %%ymm2 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" // iteration 3 + "vbroadcastsd 12 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm4 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm5 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm6 \n\t" + " \n\t" + "vbroadcastsd 13 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm7 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm8 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm9 \n\t" + " \n\t" + "vbroadcastsd 14 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm11 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm12 \n\t" + " \n\t" + "vbroadcastsd 15 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm13 \n\t" + "vmovapd 12 * 32(%%rax), %%ymm0 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm14 \n\t" + "vmovapd 13 * 32(%%rax), %%ymm1 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm15 \n\t" + "vmovapd 14 * 32(%%rax), %%ymm2 \n\t" + " \n\t" + " \n\t" + " \n\t" + "addq $4 * 12 * 8, %%rax \n\t" // a += 4*12 (unroll x mr) + "addq $4 * 4 * 8, %%rbx \n\t" // b += 4*4 (unroll x nr) + " \n\t" + " \n\t" + "decq %%rsi \n\t" // i -= 1; + "jne .DLOOPKITER \n\t" // iterate again if i != 0. + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + ".DCONSIDKLEFT: \n\t" + " \n\t" + "movq %1, %%rsi \n\t" // i = k_left; + "testq %%rsi, %%rsi \n\t" // check i via logical AND. + "je .DPOSTACCUM \n\t" // if i == 0, we're done; jump to end. + " \n\t" // else, we prepare to enter k_left loop. + " \n\t" + " \n\t" + ".DLOOPKLEFT: \n\t" // EDGE LOOP + " \n\t" + "prefetcht0 16 * 32(%%rax) \n\t" + " \n\t" + "vbroadcastsd 0 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm4 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm5 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm6 \n\t" + " \n\t" + "vbroadcastsd 1 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm7 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm8 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm9 \n\t" + " \n\t" + "vbroadcastsd 2 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm11 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm12 \n\t" + " \n\t" + "vbroadcastsd 3 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm13 \n\t" + "vmovapd 3 * 32(%%rax), %%ymm0 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm14 \n\t" + "vmovapd 4 * 32(%%rax), %%ymm1 \n\t" + "vfmadd231pd %%ymm2, %%ymm3, %%ymm15 \n\t" + "vmovapd 5 * 32(%%rax), %%ymm2 \n\t" + " \n\t" + " \n\t" + " \n\t" + "addq $1 * 12 * 8, %%rax \n\t" // a += 1*12 (unroll x mr) + "addq $1 * 4 * 8, %%rbx \n\t" // b += 1*4 (unroll x nr) + " \n\t" + " \n\t" + "decq %%rsi \n\t" // i -= 1; + "jne .DLOOPKLEFT \n\t" // iterate again if i != 0. + " \n\t" + " \n\t" + " \n\t" + ".DPOSTACCUM: \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "movq %4, %%rax \n\t" // load address of alpha + "movq %5, %%rbx \n\t" // load address of beta + "vbroadcastsd (%%rax), %%ymm0 \n\t" // load alpha and duplicate + "vbroadcastsd (%%rbx), %%ymm3 \n\t" // load beta and duplicate + " \n\t" + "vmulpd %%ymm0, %%ymm4, %%ymm4 \n\t" // scale by alpha + "vmulpd %%ymm0, %%ymm5, %%ymm5 \n\t" + "vmulpd %%ymm0, %%ymm6, %%ymm6 \n\t" + "vmulpd %%ymm0, %%ymm7, %%ymm7 \n\t" + "vmulpd %%ymm0, %%ymm8, %%ymm8 \n\t" + "vmulpd %%ymm0, %%ymm9, %%ymm9 \n\t" + "vmulpd %%ymm0, %%ymm10, %%ymm10 \n\t" + "vmulpd %%ymm0, %%ymm11, %%ymm11 \n\t" + "vmulpd %%ymm0, %%ymm12, %%ymm12 \n\t" + "vmulpd %%ymm0, %%ymm13, %%ymm13 \n\t" + "vmulpd %%ymm0, %%ymm14, %%ymm14 \n\t" + "vmulpd %%ymm0, %%ymm15, %%ymm15 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "movq %7, %%rsi \n\t" // load rs_c + "leaq (,%%rsi,8), %%rsi \n\t" // rsi = rs_c * sizeof(double) + " \n\t" + "leaq (%%rcx,%%rsi,4), %%rdx \n\t" // rdx = c + 4*rs_c; + "leaq (%%rcx,%%rsi,8), %%r12 \n\t" // r12 = c + 8*rs_c; + " \n\t" + "leaq (%%rsi,%%rsi,2), %%r13 \n\t" // r13 = 3*rs_c; + //"leaq (%%rsi,%%rsi,4), %%r15 \n\t" // r15 = 5*rs_c; + //"leaq (%%r13,%%rsi,4), %%r10 \n\t" // r10 = 7*rs_c; + " \n\t" + " \n\t" + " \n\t" + " \n\t" // determine if + " \n\t" // c % 32 == 0, AND + " \n\t" // 8*cs_c % 32 == 0, AND + " \n\t" // rs_c == 1 + " \n\t" // ie: aligned, ldim aligned, and + " \n\t" // column-stored + " \n\t" + "cmpq $8, %%rsi \n\t" // set ZF if (8*rs_c) == 8. + "sete %%bl \n\t" // bl = ( ZF == 1 ? 1 : 0 ); + "testq $31, %%rcx \n\t" // set ZF if c & 32 is zero. + "setz %%bh \n\t" // bh = ( ZF == 0 ? 1 : 0 ); + "testq $31, %%rdi \n\t" // set ZF if (8*cs_c) & 32 is zero. + "setz %%al \n\t" // al = ( ZF == 0 ? 1 : 0 ); + " \n\t" // and(bl,bh) followed by + " \n\t" // and(bh,al) will reveal result + " \n\t" + " \n\t" // now avoid loading C if beta == 0 + " \n\t" + "vxorpd %%ymm0, %%ymm0, %%ymm0 \n\t" // set ymm0 to zero. + "vucomisd %%xmm0, %%xmm3 \n\t" // set ZF if beta == 0. + "je .DBETAZERO \n\t" // if ZF = 1, jump to beta == 0 case + " \n\t" + " \n\t" + " \n\t" // check if aligned/column-stored + "andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. + "andb %%bh, %%al \n\t" // set ZF if bh & al == 1. + "jne .DCOLSTORED \n\t" // jump to column storage case + " \n\t" + " \n\t" + " \n\t" + ".DGENSTORED: \n\t" + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm4, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm7, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm10, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm13, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + //"addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "movq %%rdx, %%rcx \n\t" // rcx = c + 4*rs_c + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm5, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm8, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm11, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm14, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + //"addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "movq %%r12, %%rcx \n\t" // rcx = c + 8*rs_c + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm6, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm9, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm12, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm15, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + //"addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "jmp .DDONE \n\t" // jump to end. + " \n\t" + " \n\t" + " \n\t" + ".DCOLSTORED: \n\t" + " \n\t" + " \n\t" + "vmovapd (%%rcx), %%ymm0 \n\t" + "vfmadd213pd %%ymm4, %%ymm3, %%ymm0 \n\t" + "vmovapd %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovapd (%%rdx), %%ymm1 \n\t" + "vfmadd213pd %%ymm5, %%ymm3, %%ymm1 \n\t" + "vmovapd %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovapd (%%r12), %%ymm2 \n\t" + "vfmadd213pd %%ymm6, %%ymm3, %%ymm2 \n\t" + "vmovapd %%ymm2, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + " \n\t" + "vmovapd (%%rcx), %%ymm0 \n\t" + "vfmadd213pd %%ymm7, %%ymm3, %%ymm0 \n\t" + "vmovapd %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovapd (%%rdx), %%ymm1 \n\t" + "vfmadd213pd %%ymm8, %%ymm3, %%ymm1 \n\t" + "vmovapd %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovapd (%%r12), %%ymm2 \n\t" + "vfmadd213pd %%ymm9, %%ymm3, %%ymm2 \n\t" + "vmovapd %%ymm2, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + " \n\t" + "vmovapd (%%rcx), %%ymm0 \n\t" + "vfmadd213pd %%ymm10, %%ymm3, %%ymm0 \n\t" + "vmovapd %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovapd (%%rdx), %%ymm1 \n\t" + "vfmadd213pd %%ymm11, %%ymm3, %%ymm1 \n\t" + "vmovapd %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovapd (%%r12), %%ymm2 \n\t" + "vfmadd213pd %%ymm12, %%ymm3, %%ymm2 \n\t" + "vmovapd %%ymm2, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + " \n\t" + "vmovapd (%%rcx), %%ymm0 \n\t" + "vfmadd213pd %%ymm13, %%ymm3, %%ymm0 \n\t" + "vmovapd %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovapd (%%rdx), %%ymm1 \n\t" + "vfmadd213pd %%ymm14, %%ymm3, %%ymm1 \n\t" + "vmovapd %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovapd (%%r12), %%ymm2 \n\t" + "vfmadd213pd %%ymm15, %%ymm3, %%ymm2 \n\t" + "vmovapd %%ymm2, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "jmp .DDONE \n\t" // jump to end. + " \n\t" + " \n\t" + " \n\t" + ".DBETAZERO: \n\t" + " \n\t" // check if aligned/column-stored + "andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. + "andb %%bh, %%al \n\t" // set ZF if bh & al == 1. + "jne .DCOLSTORBZ \n\t" // jump to column storage case + " \n\t" + " \n\t" + " \n\t" + ".DGENSTORBZ: \n\t" + " \n\t" + " \n\t" + "vmovapd %%ymm4, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovapd %%ymm7, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovapd %%ymm10, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovapd %%ymm13, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + //"addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "movq %%rdx, %%rcx \n\t" // rcx = c + 4*rs_c + " \n\t" + " \n\t" + "vmovapd %%ymm5, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovapd %%ymm8, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovapd %%ymm11, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovapd %%ymm14, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + //"addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "movq %%r12, %%rcx \n\t" // rcx = c + 8*rs_c + " \n\t" + " \n\t" + "vmovapd %%ymm6, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovapd %%ymm9, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovapd %%ymm12, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovapd %%ymm15, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + //"addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "jmp .DDONE \n\t" // jump to end. + " \n\t" + " \n\t" + " \n\t" + ".DCOLSTORBZ: \n\t" + " \n\t" + " \n\t" + "vmovapd %%ymm4, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovapd %%ymm5, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovapd %%ymm6, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + "vmovapd %%ymm7, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovapd %%ymm8, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovapd %%ymm9, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + "vmovapd %%ymm10, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovapd %%ymm11, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + "vmovapd %%ymm12, (%%r12) \n\t" + "addq %%rdi, %%r12 \n\t" + " \n\t" + "vmovapd %%ymm13, (%%rcx) \n\t" + //"addq %%rdi, %%rcx \n\t" + "vmovapd %%ymm14, (%%rdx) \n\t" + //"addq %%rdi, %%rdx \n\t" + "vmovapd %%ymm15, (%%r12) \n\t" + //"addq %%rdi, %%r12 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + ".DDONE: \n\t" + " \n\t" + + : // output operands (none) + : // input operands + "m" (k_iter), // 0 + "m" (k_left), // 1 + "m" (a), // 2 + "m" (b), // 3 + "m" (alpha), // 4 + "m" (beta), // 5 + "m" (c), // 6 + "m" (rs_c), // 7 + "m" (cs_c)/*, // 8 + "m" (b_next), // 9 + "m" (a_next)*/ // 10 + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm6", "xmm7", + "xmm8", "xmm9", "xmm10", "xmm11", + "xmm12", "xmm13", "xmm14", "xmm15", + "memory" + ); +} + +#if 0 + +void bli_cgemm_asm_( + dim_t k, + scomplex* restrict alpha, + scomplex* restrict a, + scomplex* restrict b, + scomplex* restrict beta, + scomplex* restrict c, inc_t rs_c, inc_t cs_c, + auxinfo_t* data + ) +{ + //void* a_next = bli_auxinfo_next_a( data ); + //void* b_next = bli_auxinfo_next_b( data ); + + //dim_t k_iter = k / 4; + //dim_t k_left = k % 4; + +} + + + +void bli_zgemm_asm_( + dim_t k, + dcomplex* restrict alpha, + dcomplex* restrict a, + dcomplex* restrict b, + dcomplex* restrict beta, + dcomplex* restrict c, inc_t rs_c, inc_t cs_c, + auxinfo_t* data + ) +{ + //void* a_next = bli_auxinfo_next_a( data ); + //void* b_next = bli_auxinfo_next_b( data ); + + //dim_t k_iter = k / 4; + //dim_t k_left = k % 4; + +} + +#endif diff --git a/kernels/x86_64/avx2/3/bli_gemm_asm_d8x6.c b/kernels/x86_64/avx2/3/bli_gemm_asm_d8x6.c new file mode 100644 index 0000000000..47cc7ad68b --- /dev/null +++ b/kernels/x86_64/avx2/3/bli_gemm_asm_d8x6.c @@ -0,0 +1,1306 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas at Austin nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + + +#define SGEMM_INPUT_GS_BETA_NZ \ + "vmovlps (%%rcx ), %%xmm0, %%xmm0 \n\t" \ + "vmovhps (%%rcx,%%rsi,1), %%xmm0, %%xmm0 \n\t" \ + "vmovlps (%%rcx,%%rsi,2), %%xmm1, %%xmm1 \n\t" \ + "vmovhps (%%rcx,%%r13 ), %%xmm1, %%xmm1 \n\t" \ + "vshufps $0x88, %%xmm1, %%xmm0, %%xmm0 \n\t" \ + "vmovlps (%%rcx,%%rsi,4), %%xmm2, %%xmm2 \n\t" \ + "vmovhps (%%rcx,%%r15 ), %%xmm2, %%xmm2 \n\t" \ + "vmovlps (%%rcx,%%r13,2), %%xmm1, %%xmm1 \n\t" \ + "vmovhps (%%rcx,%%r10 ), %%xmm1, %%xmm1 \n\t" \ + "vshufps $0x88, %%xmm1, %%xmm2, %%xmm2 \n\t" \ + "vperm2f128 $0x20, %%ymm2, %%ymm0, %%ymm0 \n\t" + +#define SGEMM_OUTPUT_GS_BETA_NZ \ + "vextractf128 $1, %%ymm0, %%xmm2 \n\t" \ + "vmovss %%xmm0, (%%rcx ) \n\t" \ + "vpermilps $0x39, %%xmm0, %%xmm1 \n\t" \ + "vmovss %%xmm1, (%%rcx,%%rsi,1) \n\t" \ + "vpermilps $0x39, %%xmm1, %%xmm0 \n\t" \ + "vmovss %%xmm0, (%%rcx,%%rsi,2) \n\t" \ + "vpermilps $0x39, %%xmm0, %%xmm1 \n\t" \ + "vmovss %%xmm1, (%%rcx,%%r13 ) \n\t" \ + "vmovss %%xmm2, (%%rcx,%%rsi,4) \n\t" \ + "vpermilps $0x39, %%xmm2, %%xmm1 \n\t" \ + "vmovss %%xmm1, (%%rcx,%%r15 ) \n\t" \ + "vpermilps $0x39, %%xmm1, %%xmm2 \n\t" \ + "vmovss %%xmm2, (%%rcx,%%r13,2) \n\t" \ + "vpermilps $0x39, %%xmm2, %%xmm1 \n\t" \ + "vmovss %%xmm1, (%%rcx,%%r10 ) \n\t" + +void bli_sgemm_asm_16x6( + dim_t k, + float* restrict alpha, + float* restrict a, + float* restrict b, + float* restrict beta, + float* restrict c, inc_t rs_c, inc_t cs_c, + auxinfo_t* data + ) +{ + //void* a_next = bli_auxinfo_next_a( data ); + //void* b_next = bli_auxinfo_next_b( data ); + + dim_t k_iter = k / 4; + dim_t k_left = k % 4; + + __asm__ volatile + ( + " \n\t" + "vzeroall \n\t" // zero all xmm/ymm registers. + " \n\t" + " \n\t" + "movq %2, %%rax \n\t" // load address of a. + "movq %3, %%rbx \n\t" // load address of b. + //"movq %9, %%r15 \n\t" // load address of b_next. + " \n\t" + "addq $32 * 4, %%rax \n\t" + " \n\t" // initialize loop by pre-loading + "vmovaps -4 * 32(%%rax), %%ymm0 \n\t" + "vmovaps -3 * 32(%%rax), %%ymm1 \n\t" + " \n\t" + "movq %6, %%rcx \n\t" // load address of c + "movq %8, %%rdi \n\t" // load cs_c + "leaq (,%%rdi,4), %%rdi \n\t" // cs_c *= sizeof(float) + " \n\t" + "leaq (%%rdi,%%rdi,2), %%r13 \n\t" // r13 = 3*cs_c; + "leaq (%%rcx,%%r13,1), %%rdx \n\t" // rdx = c + 3*cs_c; + "prefetcht0 7 * 8(%%rcx) \n\t" // prefetch c + 0*cs_c + "prefetcht0 7 * 8(%%rcx,%%rdi) \n\t" // prefetch c + 1*cs_c + "prefetcht0 7 * 8(%%rcx,%%rdi,2) \n\t" // prefetch c + 2*cs_c + "prefetcht0 7 * 8(%%rdx) \n\t" // prefetch c + 3*cs_c + "prefetcht0 7 * 8(%%rdx,%%rdi) \n\t" // prefetch c + 4*cs_c + "prefetcht0 7 * 8(%%rdx,%%rdi,2) \n\t" // prefetch c + 5*cs_c + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "movq %0, %%rsi \n\t" // i = k_iter; + "testq %%rsi, %%rsi \n\t" // check i via logical AND. + "je .SCONSIDKLEFT \n\t" // if i == 0, jump to code that + " \n\t" // contains the k_left loop. + " \n\t" + " \n\t" + ".SLOOPKITER: \n\t" // MAIN LOOP + " \n\t" + " \n\t" + " \n\t" // iteration 0 + "prefetcht0 16 * 32(%%rax) \n\t" + " \n\t" + "vbroadcastss 0 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 1 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm4 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm5 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm6 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm7 \n\t" + " \n\t" + "vbroadcastss 2 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 3 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm8 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm9 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm11 \n\t" + " \n\t" + "vbroadcastss 4 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 5 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm12 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm13 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm14 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm15 \n\t" + " \n\t" + "vmovaps -2 * 32(%%rax), %%ymm0 \n\t" + "vmovaps -1 * 32(%%rax), %%ymm1 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" // iteration 1 + "vbroadcastss 6 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 7 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm4 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm5 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm6 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm7 \n\t" + " \n\t" + "vbroadcastss 8 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 9 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm8 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm9 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm11 \n\t" + " \n\t" + "vbroadcastss 10 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 11 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm12 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm13 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm14 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm15 \n\t" + " \n\t" + "vmovaps 0 * 32(%%rax), %%ymm0 \n\t" + "vmovaps 1 * 32(%%rax), %%ymm1 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" // iteration 2 + "prefetcht0 20 * 32(%%rax) \n\t" + " \n\t" + "vbroadcastss 12 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 13 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm4 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm5 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm6 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm7 \n\t" + " \n\t" + "vbroadcastss 14 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 15 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm8 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm9 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm11 \n\t" + " \n\t" + "vbroadcastss 16 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 17 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm12 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm13 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm14 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm15 \n\t" + " \n\t" + "vmovaps 2 * 32(%%rax), %%ymm0 \n\t" + "vmovaps 3 * 32(%%rax), %%ymm1 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" // iteration 3 + "vbroadcastss 18 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 19 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm4 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm5 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm6 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm7 \n\t" + " \n\t" + "vbroadcastss 20 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 21 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm8 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm9 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm11 \n\t" + " \n\t" + "vbroadcastss 22 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 23 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm12 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm13 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm14 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm15 \n\t" + " \n\t" + "addq $4 * 16 * 4, %%rax \n\t" // a += 4*16 (unroll x mr) + "addq $4 * 6 * 4, %%rbx \n\t" // b += 4*6 (unroll x nr) + " \n\t" + "vmovaps -4 * 32(%%rax), %%ymm0 \n\t" + "vmovaps -3 * 32(%%rax), %%ymm1 \n\t" + " \n\t" + " \n\t" + "decq %%rsi \n\t" // i -= 1; + "jne .SLOOPKITER \n\t" // iterate again if i != 0. + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + ".SCONSIDKLEFT: \n\t" + " \n\t" + "movq %1, %%rsi \n\t" // i = k_left; + "testq %%rsi, %%rsi \n\t" // check i via logical AND. + "je .SPOSTACCUM \n\t" // if i == 0, we're done; jump to end. + " \n\t" // else, we prepare to enter k_left loop. + " \n\t" + " \n\t" + ".SLOOPKLEFT: \n\t" // EDGE LOOP + " \n\t" + "prefetcht0 16 * 32(%%rax) \n\t" + " \n\t" + "vbroadcastss 0 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 1 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm4 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm5 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm6 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm7 \n\t" + " \n\t" + "vbroadcastss 2 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 3 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm8 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm9 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm11 \n\t" + " \n\t" + "vbroadcastss 4 * 4(%%rbx), %%ymm2 \n\t" + "vbroadcastss 5 * 4(%%rbx), %%ymm3 \n\t" + "vfmadd231ps %%ymm0, %%ymm2, %%ymm12 \n\t" + "vfmadd231ps %%ymm1, %%ymm2, %%ymm13 \n\t" + "vfmadd231ps %%ymm0, %%ymm3, %%ymm14 \n\t" + "vfmadd231ps %%ymm1, %%ymm3, %%ymm15 \n\t" + " \n\t" + "addq $1 * 16 * 4, %%rax \n\t" // a += 1*16 (unroll x mr) + "addq $1 * 6 * 4, %%rbx \n\t" // b += 1*6 (unroll x nr) + " \n\t" + "vmovaps -4 * 32(%%rax), %%ymm0 \n\t" + "vmovaps -3 * 32(%%rax), %%ymm1 \n\t" + " \n\t" + " \n\t" + "decq %%rsi \n\t" // i -= 1; + "jne .SLOOPKLEFT \n\t" // iterate again if i != 0. + " \n\t" + " \n\t" + " \n\t" + ".SPOSTACCUM: \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "movq %4, %%rax \n\t" // load address of alpha + "movq %5, %%rbx \n\t" // load address of beta + "vbroadcastss (%%rax), %%ymm0 \n\t" // load alpha and duplicate + "vbroadcastss (%%rbx), %%ymm3 \n\t" // load beta and duplicate + " \n\t" + "vmulps %%ymm0, %%ymm4, %%ymm4 \n\t" // scale by alpha + "vmulps %%ymm0, %%ymm5, %%ymm5 \n\t" + "vmulps %%ymm0, %%ymm6, %%ymm6 \n\t" + "vmulps %%ymm0, %%ymm7, %%ymm7 \n\t" + "vmulps %%ymm0, %%ymm8, %%ymm8 \n\t" + "vmulps %%ymm0, %%ymm9, %%ymm9 \n\t" + "vmulps %%ymm0, %%ymm10, %%ymm10 \n\t" + "vmulps %%ymm0, %%ymm11, %%ymm11 \n\t" + "vmulps %%ymm0, %%ymm12, %%ymm12 \n\t" + "vmulps %%ymm0, %%ymm13, %%ymm13 \n\t" + "vmulps %%ymm0, %%ymm14, %%ymm14 \n\t" + "vmulps %%ymm0, %%ymm15, %%ymm15 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "movq %7, %%rsi \n\t" // load rs_c + "leaq (,%%rsi,4), %%rsi \n\t" // rsi = rs_c * sizeof(float) + " \n\t" + "leaq (%%rcx,%%rsi,8), %%rdx \n\t" // load address of c + 8*rs_c; + " \n\t" + "leaq (%%rsi,%%rsi,2), %%r13 \n\t" // r13 = 3*rs_c; + "leaq (%%rsi,%%rsi,4), %%r15 \n\t" // r15 = 5*rs_c; + "leaq (%%r13,%%rsi,4), %%r10 \n\t" // r10 = 7*rs_c; + " \n\t" + " \n\t" + " \n\t" + " \n\t" // determine if + " \n\t" // c % 32 == 0, AND + " \n\t" // 4*cs_c % 32 == 0, AND + " \n\t" // rs_c == 1 + " \n\t" // ie: aligned, ldim aligned, and + " \n\t" // column-stored + " \n\t" + "cmpq $4, %%rsi \n\t" // set ZF if (4*rs_c) == 4. + "sete %%bl \n\t" // bl = ( ZF == 1 ? 1 : 0 ); + "testq $31, %%rcx \n\t" // set ZF if c & 32 is zero. + "setz %%bh \n\t" // bh = ( ZF == 0 ? 1 : 0 ); + "testq $31, %%rdi \n\t" // set ZF if (4*cs_c) & 32 is zero. + "setz %%al \n\t" // al = ( ZF == 0 ? 1 : 0 ); + " \n\t" // and(bl,bh) followed by + " \n\t" // and(bh,al) will reveal result + " \n\t" + " \n\t" // now avoid loading C if beta == 0 + " \n\t" + "vxorps %%ymm0, %%ymm0, %%ymm0 \n\t" // set ymm0 to zero. + "vucomiss %%xmm0, %%xmm3 \n\t" // set ZF if beta == 0. + "je .SBETAZERO \n\t" // if ZF = 1, jump to beta == 0 case + " \n\t" + " \n\t" + " \n\t" // check if aligned/column-stored + "andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. + "andb %%bh, %%al \n\t" // set ZF if bh & al == 1. + "jne .SCOLSTORED \n\t" // jump to column storage case + " \n\t" + " \n\t" + " \n\t" + ".SGENSTORED: \n\t" + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm4, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm6, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm8, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm10, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm12, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm14, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + " \n\t" + " \n\t" + "movq %%rdx, %%rcx \n\t" // rcx = c + 8*rs_c + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm5, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm7, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm9, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm11, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm13, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + SGEMM_INPUT_GS_BETA_NZ + "vfmadd213ps %%ymm15, %%ymm3, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + " \n\t" + " \n\t" + " \n\t" + "jmp .SDONE \n\t" // jump to end. + " \n\t" + " \n\t" + " \n\t" + ".SCOLSTORED: \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213ps %%ymm4, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213ps %%ymm5, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213ps %%ymm6, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213ps %%ymm7, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213ps %%ymm8, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213ps %%ymm9, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213ps %%ymm10, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213ps %%ymm11, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213ps %%ymm12, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213ps %%ymm13, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213ps %%ymm14, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213ps %%ymm15, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + " \n\t" + "jmp .SDONE \n\t" // jump to end. + " \n\t" + " \n\t" + " \n\t" + ".SBETAZERO: \n\t" + " \n\t" // check if aligned/column-stored + "andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. + "andb %%bh, %%al \n\t" // set ZF if bh & al == 1. + "jne .SCOLSTORBZ \n\t" // jump to column storage case + " \n\t" + " \n\t" + " \n\t" + ".SGENSTORBZ: \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm4, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm6, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm8, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm10, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm12, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm14, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + " \n\t" + " \n\t" + "movq %%rdx, %%rcx \n\t" // rcx = c + 8*rs_c + " \n\t" + " \n\t" + "vmovaps %%ymm5, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm7, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm9, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm11, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm13, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm15, %%ymm0 \n\t" + SGEMM_OUTPUT_GS_BETA_NZ + " \n\t" + " \n\t" + " \n\t" + "jmp .SDONE \n\t" // jump to end. + " \n\t" + " \n\t" + " \n\t" + ".SCOLSTORBZ: \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm4, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm5, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + "vmovaps %%ymm6, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm7, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm8, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm9, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm10, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm11, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm12, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm13, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm14, (%%rcx) \n\t" + " \n\t" + "vmovaps %%ymm15, (%%rdx) \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + ".SDONE: \n\t" + " \n\t" + + : // output operands (none) + : // input operands + "m" (k_iter), // 0 + "m" (k_left), // 1 + "m" (a), // 2 + "m" (b), // 3 + "m" (alpha), // 4 + "m" (beta), // 5 + "m" (c), // 6 + "m" (rs_c), // 7 + "m" (cs_c)/*, // 8 + "m" (b_next), // 9 + "m" (a_next)*/ // 10 + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm6", "xmm7", + "xmm8", "xmm9", "xmm10", "xmm11", + "xmm12", "xmm13", "xmm14", "xmm15", + "memory" + ); +} + +#define DGEMM_INPUT_GS_BETA_NZ \ + "vmovlpd (%%rcx ), %%xmm0, %%xmm0 \n\t" \ + "vmovhpd (%%rcx,%%rsi,1), %%xmm0, %%xmm0 \n\t" \ + "vmovlpd (%%rcx,%%rsi,2), %%xmm1, %%xmm1 \n\t" \ + "vmovhpd (%%rcx,%%r13 ), %%xmm1, %%xmm1 \n\t" \ + "vperm2f128 $0x20, %%ymm1, %%ymm0, %%ymm0 \n\t" /*\ + "vmovlps (%%rcx,%%rsi,4), %%xmm2, %%xmm2 \n\t" \ + "vmovhps (%%rcx,%%r15 ), %%xmm2, %%xmm2 \n\t" \ + "vmovlps (%%rcx,%%r13,2), %%xmm1, %%xmm1 \n\t" \ + "vmovhps (%%rcx,%%r10 ), %%xmm1, %%xmm1 \n\t" \ + "vperm2f128 $0x20, %%ymm1, %%ymm2, %%ymm2 \n\t"*/ + +#define DGEMM_OUTPUT_GS_BETA_NZ \ + "vextractf128 $1, %%ymm0, %%xmm1 \n\t" \ + "vmovlpd %%xmm0, (%%rcx ) \n\t" \ + "vmovhpd %%xmm0, (%%rcx,%%rsi ) \n\t" \ + "vmovlpd %%xmm1, (%%rcx,%%rsi,2) \n\t" \ + "vmovhpd %%xmm1, (%%rcx,%%r13 ) \n\t" /*\ + "vextractf128 $1, %%ymm2, %%xmm1 \n\t" \ + "vmovlpd %%xmm2, (%%rcx,%%rsi,4) \n\t" \ + "vmovhpd %%xmm2, (%%rcx,%%r15 ) \n\t" \ + "vmovlpd %%xmm1, (%%rcx,%%r13,2) \n\t" \ + "vmovhpd %%xmm1, (%%rcx,%%r10 ) \n\t"*/ + +void bli_dgemm_asm_8x6( + dim_t k, + double* restrict alpha, + double* restrict a, + double* restrict b, + double* restrict beta, + double* restrict c, inc_t rs_c, inc_t cs_c, + auxinfo_t* data + ) +{ + //void* a_next = bli_auxinfo_next_a( data ); + //void* b_next = bli_auxinfo_next_b( data ); + + dim_t k_iter = k / 4; + dim_t k_left = k % 4; + + __asm__ volatile + ( + " \n\t" + "vzeroall \n\t" // zero all xmm/ymm registers. + " \n\t" + " \n\t" + "movq %2, %%rax \n\t" // load address of a. + "movq %3, %%rbx \n\t" // load address of b. + //"movq %9, %%r15 \n\t" // load address of b_next. + " \n\t" + "addq $32 * 4, %%rax \n\t" + " \n\t" // initialize loop by pre-loading + "vmovaps -4 * 32(%%rax), %%ymm0 \n\t" + "vmovaps -3 * 32(%%rax), %%ymm1 \n\t" + " \n\t" + "movq %6, %%rcx \n\t" // load address of c + "movq %8, %%rdi \n\t" // load cs_c + "leaq (,%%rdi,8), %%rdi \n\t" // cs_c *= sizeof(double) + " \n\t" + "leaq (%%rdi,%%rdi,2), %%r13 \n\t" // r13 = 3*cs_c; + "leaq (%%rcx,%%r13,1), %%rdx \n\t" // rdx = c + 3*cs_c; + "prefetcht0 7 * 8(%%rcx) \n\t" // prefetch c + 0*cs_c + "prefetcht0 7 * 8(%%rcx,%%rdi) \n\t" // prefetch c + 1*cs_c + "prefetcht0 7 * 8(%%rcx,%%rdi,2) \n\t" // prefetch c + 2*cs_c + "prefetcht0 7 * 8(%%rdx) \n\t" // prefetch c + 3*cs_c + "prefetcht0 7 * 8(%%rdx,%%rdi) \n\t" // prefetch c + 4*cs_c + "prefetcht0 7 * 8(%%rdx,%%rdi,2) \n\t" // prefetch c + 5*cs_c + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "movq %0, %%rsi \n\t" // i = k_iter; + "testq %%rsi, %%rsi \n\t" // check i via logical AND. + "je .DCONSIDKLEFT \n\t" // if i == 0, jump to code that + " \n\t" // contains the k_left loop. + " \n\t" + " \n\t" + ".DLOOPKITER: \n\t" // MAIN LOOP + " \n\t" + " \n\t" + " \n\t" // iteration 0 + "prefetcht0 16 * 32(%%rax) \n\t" + " \n\t" + "vbroadcastsd 0 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 1 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm4 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm5 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm6 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm7 \n\t" + " \n\t" + "vbroadcastsd 2 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 3 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm8 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm9 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm11 \n\t" + " \n\t" + "vbroadcastsd 4 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 5 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm12 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm13 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm14 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm15 \n\t" + " \n\t" + "vmovaps -2 * 32(%%rax), %%ymm0 \n\t" + "vmovaps -1 * 32(%%rax), %%ymm1 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" // iteration 1 + "vbroadcastsd 6 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 7 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm4 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm5 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm6 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm7 \n\t" + " \n\t" + "vbroadcastsd 8 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 9 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm8 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm9 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm11 \n\t" + " \n\t" + "vbroadcastsd 10 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 11 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm12 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm13 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm14 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm15 \n\t" + " \n\t" + "vmovaps 0 * 32(%%rax), %%ymm0 \n\t" + "vmovaps 1 * 32(%%rax), %%ymm1 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" // iteration 2 + "prefetcht0 20 * 32(%%rax) \n\t" + " \n\t" + "vbroadcastsd 12 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 13 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm4 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm5 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm6 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm7 \n\t" + " \n\t" + "vbroadcastsd 14 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 15 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm8 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm9 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm11 \n\t" + " \n\t" + "vbroadcastsd 16 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 17 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm12 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm13 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm14 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm15 \n\t" + " \n\t" + "vmovaps 2 * 32(%%rax), %%ymm0 \n\t" + "vmovaps 3 * 32(%%rax), %%ymm1 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" // iteration 3 + "vbroadcastsd 18 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 19 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm4 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm5 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm6 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm7 \n\t" + " \n\t" + "vbroadcastsd 20 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 21 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm8 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm9 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm11 \n\t" + " \n\t" + "vbroadcastsd 22 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 23 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm12 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm13 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm14 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm15 \n\t" + " \n\t" + "addq $4 * 8 * 8, %%rax \n\t" // a += 4*8 (unroll x mr) + "addq $4 * 6 * 8, %%rbx \n\t" // b += 4*6 (unroll x nr) + " \n\t" + "vmovaps -4 * 32(%%rax), %%ymm0 \n\t" + "vmovaps -3 * 32(%%rax), %%ymm1 \n\t" + " \n\t" + " \n\t" + "decq %%rsi \n\t" // i -= 1; + "jne .DLOOPKITER \n\t" // iterate again if i != 0. + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + ".DCONSIDKLEFT: \n\t" + " \n\t" + "movq %1, %%rsi \n\t" // i = k_left; + "testq %%rsi, %%rsi \n\t" // check i via logical AND. + "je .DPOSTACCUM \n\t" // if i == 0, we're done; jump to end. + " \n\t" // else, we prepare to enter k_left loop. + " \n\t" + " \n\t" + ".DLOOPKLEFT: \n\t" // EDGE LOOP + " \n\t" + "prefetcht0 16 * 32(%%rax) \n\t" + " \n\t" + "vbroadcastsd 0 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 1 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm4 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm5 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm6 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm7 \n\t" + " \n\t" + "vbroadcastsd 2 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 3 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm8 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm9 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm10 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm11 \n\t" + " \n\t" + "vbroadcastsd 4 * 8(%%rbx), %%ymm2 \n\t" + "vbroadcastsd 5 * 8(%%rbx), %%ymm3 \n\t" + "vfmadd231pd %%ymm0, %%ymm2, %%ymm12 \n\t" + "vfmadd231pd %%ymm1, %%ymm2, %%ymm13 \n\t" + "vfmadd231pd %%ymm0, %%ymm3, %%ymm14 \n\t" + "vfmadd231pd %%ymm1, %%ymm3, %%ymm15 \n\t" + " \n\t" + "addq $1 * 8 * 8, %%rax \n\t" // a += 1*8 (unroll x mr) + "addq $1 * 6 * 8, %%rbx \n\t" // b += 1*6 (unroll x nr) + " \n\t" + "vmovaps -4 * 32(%%rax), %%ymm0 \n\t" + "vmovaps -3 * 32(%%rax), %%ymm1 \n\t" + " \n\t" + " \n\t" + "decq %%rsi \n\t" // i -= 1; + "jne .DLOOPKLEFT \n\t" // iterate again if i != 0. + " \n\t" + " \n\t" + " \n\t" + ".DPOSTACCUM: \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "movq %4, %%rax \n\t" // load address of alpha + "movq %5, %%rbx \n\t" // load address of beta + "vbroadcastsd (%%rax), %%ymm0 \n\t" // load alpha and duplicate + "vbroadcastsd (%%rbx), %%ymm3 \n\t" // load beta and duplicate + " \n\t" + "vmulpd %%ymm0, %%ymm4, %%ymm4 \n\t" // scale by alpha + "vmulpd %%ymm0, %%ymm5, %%ymm5 \n\t" + "vmulpd %%ymm0, %%ymm6, %%ymm6 \n\t" + "vmulpd %%ymm0, %%ymm7, %%ymm7 \n\t" + "vmulpd %%ymm0, %%ymm8, %%ymm8 \n\t" + "vmulpd %%ymm0, %%ymm9, %%ymm9 \n\t" + "vmulpd %%ymm0, %%ymm10, %%ymm10 \n\t" + "vmulpd %%ymm0, %%ymm11, %%ymm11 \n\t" + "vmulpd %%ymm0, %%ymm12, %%ymm12 \n\t" + "vmulpd %%ymm0, %%ymm13, %%ymm13 \n\t" + "vmulpd %%ymm0, %%ymm14, %%ymm14 \n\t" + "vmulpd %%ymm0, %%ymm15, %%ymm15 \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "movq %7, %%rsi \n\t" // load rs_c + "leaq (,%%rsi,8), %%rsi \n\t" // rsi = rs_c * sizeof(double) + " \n\t" + "leaq (%%rcx,%%rsi,4), %%rdx \n\t" // load address of c + 4*rs_c; + " \n\t" + "leaq (%%rsi,%%rsi,2), %%r13 \n\t" // r13 = 3*rs_c; + //"leaq (%%rsi,%%rsi,4), %%r15 \n\t" // r15 = 5*rs_c; + //"leaq (%%r13,%%rsi,4), %%r10 \n\t" // r10 = 7*rs_c; + " \n\t" + " \n\t" + " \n\t" + " \n\t" // determine if + " \n\t" // c % 32 == 0, AND + " \n\t" // 8*cs_c % 32 == 0, AND + " \n\t" // rs_c == 1 + " \n\t" // ie: aligned, ldim aligned, and + " \n\t" // column-stored + " \n\t" + "cmpq $8, %%rsi \n\t" // set ZF if (8*rs_c) == 8. + "sete %%bl \n\t" // bl = ( ZF == 1 ? 1 : 0 ); + "testq $31, %%rcx \n\t" // set ZF if c & 32 is zero. + "setz %%bh \n\t" // bh = ( ZF == 0 ? 1 : 0 ); + "testq $31, %%rdi \n\t" // set ZF if (8*cs_c) & 32 is zero. + "setz %%al \n\t" // al = ( ZF == 0 ? 1 : 0 ); + " \n\t" // and(bl,bh) followed by + " \n\t" // and(bh,al) will reveal result + " \n\t" + " \n\t" // now avoid loading C if beta == 0 + " \n\t" + "vxorpd %%ymm0, %%ymm0, %%ymm0 \n\t" // set ymm0 to zero. + "vucomisd %%xmm0, %%xmm3 \n\t" // set ZF if beta == 0. + "je .DBETAZERO \n\t" // if ZF = 1, jump to beta == 0 case + " \n\t" + " \n\t" + " \n\t" // check if aligned/column-stored + "andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. + "andb %%bh, %%al \n\t" // set ZF if bh & al == 1. + "jne .DCOLSTORED \n\t" // jump to column storage case + " \n\t" + " \n\t" + " \n\t" + ".DGENSTORED: \n\t" + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm4, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm6, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm8, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm10, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm12, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm14, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + " \n\t" + " \n\t" + "movq %%rdx, %%rcx \n\t" // rcx = c + 4*rs_c + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm5, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm7, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm9, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm11, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm13, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + DGEMM_INPUT_GS_BETA_NZ + "vfmadd213pd %%ymm15, %%ymm3, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + " \n\t" + " \n\t" + " \n\t" + "jmp .DDONE \n\t" // jump to end. + " \n\t" + " \n\t" + " \n\t" + ".DCOLSTORED: \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213pd %%ymm4, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213pd %%ymm5, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213pd %%ymm6, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213pd %%ymm7, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213pd %%ymm8, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213pd %%ymm9, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213pd %%ymm10, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213pd %%ymm11, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213pd %%ymm12, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213pd %%ymm13, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps (%%rcx), %%ymm0 \n\t" + "vfmadd213pd %%ymm14, %%ymm3, %%ymm0 \n\t" + "vmovaps %%ymm0, (%%rcx) \n\t" + //"addq %%rdi, %%rcx \n\t" + "vmovaps (%%rdx), %%ymm1 \n\t" + "vfmadd213pd %%ymm15, %%ymm3, %%ymm1 \n\t" + "vmovaps %%ymm1, (%%rdx) \n\t" + //"addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + " \n\t" + "jmp .DDONE \n\t" // jump to end. + " \n\t" + " \n\t" + " \n\t" + ".DBETAZERO: \n\t" + " \n\t" // check if aligned/column-stored + "andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. + "andb %%bh, %%al \n\t" // set ZF if bh & al == 1. + "jne .DCOLSTORBZ \n\t" // jump to column storage case + " \n\t" + " \n\t" + " \n\t" + ".DGENSTORBZ: \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm4, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm6, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm8, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm10, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm12, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm14, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + " \n\t" + " \n\t" + "movq %%rdx, %%rcx \n\t" // rcx = c + 4*rs_c + " \n\t" + " \n\t" + "vmovaps %%ymm5, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm7, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm9, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm11, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm13, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + "addq %%rdi, %%rcx \n\t" // c += cs_c; + " \n\t" + " \n\t" + "vmovaps %%ymm15, %%ymm0 \n\t" + DGEMM_OUTPUT_GS_BETA_NZ + " \n\t" + " \n\t" + " \n\t" + "jmp .DDONE \n\t" // jump to end. + " \n\t" + " \n\t" + " \n\t" + ".DCOLSTORBZ: \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm4, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm5, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + "vmovaps %%ymm6, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm7, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm8, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm9, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm10, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm11, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm12, (%%rcx) \n\t" + "addq %%rdi, %%rcx \n\t" + "vmovaps %%ymm13, (%%rdx) \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "vmovaps %%ymm14, (%%rcx) \n\t" + " \n\t" + "vmovaps %%ymm15, (%%rdx) \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + ".DDONE: \n\t" + " \n\t" + + : // output operands (none) + : // input operands + "m" (k_iter), // 0 + "m" (k_left), // 1 + "m" (a), // 2 + "m" (b), // 3 + "m" (alpha), // 4 + "m" (beta), // 5 + "m" (c), // 6 + "m" (rs_c), // 7 + "m" (cs_c)/*, // 8 + "m" (b_next), // 9 + "m" (a_next)*/ // 10 + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm6", "xmm7", + "xmm8", "xmm9", "xmm10", "xmm11", + "xmm12", "xmm13", "xmm14", "xmm15", + "memory" + ); +} + +#if 0 + +void bli_cgemm_asm_( + dim_t k, + scomplex* restrict alpha, + scomplex* restrict a, + scomplex* restrict b, + scomplex* restrict beta, + scomplex* restrict c, inc_t rs_c, inc_t cs_c, + auxinfo_t* data + ) +{ + //void* a_next = bli_auxinfo_next_a( data ); + //void* b_next = bli_auxinfo_next_b( data ); + + //dim_t k_iter = k / 4; + //dim_t k_left = k % 4; + +} + + + +void bli_zgemm_asm_( + dim_t k, + dcomplex* restrict alpha, + dcomplex* restrict a, + dcomplex* restrict b, + dcomplex* restrict beta, + dcomplex* restrict c, inc_t rs_c, inc_t cs_c, + auxinfo_t* data + ) +{ + //void* a_next = bli_auxinfo_next_a( data ); + //void* b_next = bli_auxinfo_next_b( data ); + + //dim_t k_iter = k / 4; + //dim_t k_left = k % 4; + +} + +#endif diff --git a/kernels/x86_64/core2-sse3/3/bli_gemm_opt_d4x4.c b/kernels/x86_64/core2-sse3/3/bli_gemm_opt_d4x4.c index 8b2c3974ff..b4dfb1ce9b 100644 --- a/kernels/x86_64/core2-sse3/3/bli_gemm_opt_d4x4.c +++ b/kernels/x86_64/core2-sse3/3/bli_gemm_opt_d4x4.c @@ -73,23 +73,23 @@ void bli_sgemm_opt_8x4( " \n\t" "prefetcht2 0 * 4(%%r9) \n\t" // prefetch b_next " \n\t" - "xorpd %%xmm3, %%xmm3 \n\t" - "xorpd %%xmm4, %%xmm4 \n\t" - "xorpd %%xmm5, %%xmm5 \n\t" - "xorpd %%xmm6, %%xmm6 \n\t" - " \n\t" - "prefetcht0 6 * 4(%%rcx) \n\t" // prefetch c + 0*cs_c - "xorpd %%xmm8, %%xmm8 \n\t" - "movaps %%xmm8, %%xmm9 \n\t" - "prefetcht0 6 * 4(%%rcx,%%rdi) \n\t" // prefetch c + 1*cs_c - "movaps %%xmm8, %%xmm10 \n\t" - "movaps %%xmm8, %%xmm11 \n\t" - "prefetcht0 6 * 4(%%r10) \n\t" // prefetch c + 2*cs_c - "movaps %%xmm8, %%xmm12 \n\t" - "movaps %%xmm8, %%xmm13 \n\t" - "prefetcht0 6 * 4(%%r10,%%rdi) \n\t" // prefetch c + 3*cs_c - "movaps %%xmm8, %%xmm14 \n\t" - "movaps %%xmm8, %%xmm15 \n\t" + "xorps %%xmm3, %%xmm3 \n\t" + "xorps %%xmm4, %%xmm4 \n\t" + "xorps %%xmm5, %%xmm5 \n\t" + "xorps %%xmm6, %%xmm6 \n\t" + " \n\t" + "prefetcht2 6 * 4(%%rcx) \n\t" // prefetch c + 0*cs_c + "xorps %%xmm8, %%xmm8 \n\t" + "xorps %%xmm9, %%xmm9 \n\t" + "prefetcht2 6 * 4(%%rcx,%%rdi) \n\t" // prefetch c + 1*cs_c + "xorps %%xmm10, %%xmm10 \n\t" + "xorps %%xmm11, %%xmm11 \n\t" + "prefetcht2 6 * 4(%%r10) \n\t" // prefetch c + 2*cs_c + "xorps %%xmm12, %%xmm12 \n\t" + "xorps %%xmm13, %%xmm13 \n\t" + "prefetcht2 6 * 4(%%r10,%%rdi) \n\t" // prefetch c + 3*cs_c + "xorps %%xmm14, %%xmm14 \n\t" + "xorps %%xmm15, %%xmm15 \n\t" " \n\t" " \n\t" " \n\t" @@ -646,8 +646,119 @@ void bli_sgemm_opt_8x4( " \n\t" ".SGENSTORBZ: \n\t" " \n\t" - "jmp .SGENSTORED \n\t" // use gen-stored beta != 0 case for now - //"jmp .SDONE \n\t" // jump to end. + "mulps %%xmm6, %%xmm8 \n\t" // scale by alpha, + "movaps %%xmm8, %%xmm0 \n\t" + " \n\t" + "movss %%xmm0, (%%rcx ) \n\t" // and store back to memory. + "pshufd $0x39, %%xmm0, %%xmm1 \n\t" + "movss %%xmm1, (%%rcx,%%rsi,1) \n\t" + "pshufd $0x39, %%xmm1, %%xmm2 \n\t" + "movss %%xmm2, (%%rcx,%%rsi,2) \n\t" + "pshufd $0x39, %%xmm2, %%xmm3 \n\t" + "movss %%xmm3, (%%rcx,%%r11 ) \n\t" + " \n\t" + "addq %%rdi, %%rcx \n\t" + " \n\t" + " \n\t" + "mulps %%xmm6, %%xmm12 \n\t" // scale by alpha, + "movaps %%xmm12, %%xmm0 \n\t" + " \n\t" + "movss %%xmm0, (%%rdx ) \n\t" // and store back to memory. + "pshufd $0x39, %%xmm0, %%xmm1 \n\t" + "movss %%xmm1, (%%rdx,%%rsi,1) \n\t" + "pshufd $0x39, %%xmm1, %%xmm2 \n\t" + "movss %%xmm2, (%%rdx,%%rsi,2) \n\t" + "pshufd $0x39, %%xmm2, %%xmm3 \n\t" + "movss %%xmm3, (%%rdx,%%r11 ) \n\t" + " \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "mulps %%xmm6, %%xmm9 \n\t" // scale by alpha, + "movaps %%xmm9, %%xmm0 \n\t" + " \n\t" + "movss %%xmm0, (%%rcx ) \n\t" // and store back to memory. + "pshufd $0x39, %%xmm0, %%xmm1 \n\t" + "movss %%xmm1, (%%rcx,%%rsi,1) \n\t" + "pshufd $0x39, %%xmm1, %%xmm2 \n\t" + "movss %%xmm2, (%%rcx,%%rsi,2) \n\t" + "pshufd $0x39, %%xmm2, %%xmm3 \n\t" + "movss %%xmm3, (%%rcx,%%r11 ) \n\t" + " \n\t" + "addq %%rdi, %%rcx \n\t" + " \n\t" + " \n\t" + "mulps %%xmm6, %%xmm13 \n\t" // scale by alpha, + "movaps %%xmm13, %%xmm0 \n\t" + " \n\t" + "movss %%xmm0, (%%rdx ) \n\t" // and store back to memory. + "pshufd $0x39, %%xmm0, %%xmm1 \n\t" + "movss %%xmm1, (%%rdx,%%rsi,1) \n\t" + "pshufd $0x39, %%xmm1, %%xmm2 \n\t" + "movss %%xmm2, (%%rdx,%%rsi,2) \n\t" + "pshufd $0x39, %%xmm2, %%xmm3 \n\t" + "movss %%xmm3, (%%rdx,%%r11 ) \n\t" + " \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "mulps %%xmm6, %%xmm10 \n\t" // scale by alpha, + "movaps %%xmm10, %%xmm0 \n\t" + " \n\t" + "movss %%xmm0, (%%rcx ) \n\t" // and store back to memory. + "pshufd $0x39, %%xmm0, %%xmm1 \n\t" + "movss %%xmm1, (%%rcx,%%rsi,1) \n\t" + "pshufd $0x39, %%xmm1, %%xmm2 \n\t" + "movss %%xmm2, (%%rcx,%%rsi,2) \n\t" + "pshufd $0x39, %%xmm2, %%xmm3 \n\t" + "movss %%xmm3, (%%rcx,%%r11 ) \n\t" + " \n\t" + "addq %%rdi, %%rcx \n\t" + " \n\t" + " \n\t" + "mulps %%xmm6, %%xmm14 \n\t" // scale by alpha, + "movaps %%xmm14, %%xmm0 \n\t" + " \n\t" + "movss %%xmm0, (%%rdx ) \n\t" // and store back to memory. + "pshufd $0x39, %%xmm0, %%xmm1 \n\t" + "movss %%xmm1, (%%rdx,%%rsi,1) \n\t" + "pshufd $0x39, %%xmm1, %%xmm2 \n\t" + "movss %%xmm2, (%%rdx,%%rsi,2) \n\t" + "pshufd $0x39, %%xmm2, %%xmm3 \n\t" + "movss %%xmm3, (%%rdx,%%r11 ) \n\t" + " \n\t" + "addq %%rdi, %%rdx \n\t" + " \n\t" + " \n\t" + "mulps %%xmm6, %%xmm11 \n\t" // scale by alpha, + "movaps %%xmm11, %%xmm0 \n\t" + " \n\t" + "movss %%xmm0, (%%rcx ) \n\t" // and store back to memory. + "pshufd $0x39, %%xmm0, %%xmm1 \n\t" + "movss %%xmm1, (%%rcx,%%rsi,1) \n\t" + "pshufd $0x39, %%xmm1, %%xmm2 \n\t" + "movss %%xmm2, (%%rcx,%%rsi,2) \n\t" + "pshufd $0x39, %%xmm2, %%xmm3 \n\t" + "movss %%xmm3, (%%rcx,%%r11 ) \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "mulps %%xmm6, %%xmm15 \n\t" // scale by alpha, + "movaps %%xmm15, %%xmm0 \n\t" + " \n\t" + "movss %%xmm0, (%%rdx ) \n\t" // and store back to memory. + "pshufd $0x39, %%xmm0, %%xmm1 \n\t" + "movss %%xmm1, (%%rdx,%%rsi,1) \n\t" + "pshufd $0x39, %%xmm1, %%xmm2 \n\t" + "movss %%xmm2, (%%rdx,%%rsi,2) \n\t" + "pshufd $0x39, %%xmm2, %%xmm3 \n\t" + "movss %%xmm3, (%%rdx,%%r11 ) \n\t" + " \n\t" + " \n\t" + " \n\t" + " \n\t" + "jmp .SDONE \n\t" // jump to end. " \n\t" " \n\t" " \n\t" @@ -770,16 +881,16 @@ void bli_dgemm_opt_4x4( " \n\t" "prefetcht2 3 * 8(%%rcx) \n\t" // prefetch c + 0*cs_c "xorpd %%xmm8, %%xmm8 \n\t" - "movaps %%xmm8, %%xmm9 \n\t" + "xorpd %%xmm9, %%xmm9 \n\t" "prefetcht2 3 * 8(%%rcx,%%rdi) \n\t" // prefetch c + 1*cs_c - "movaps %%xmm8, %%xmm10 \n\t" - "movaps %%xmm8, %%xmm11 \n\t" + "xorpd %%xmm10, %%xmm10 \n\t" + "xorpd %%xmm11, %%xmm11 \n\t" "prefetcht2 3 * 8(%%r10) \n\t" // prefetch c + 2*cs_c - "movaps %%xmm8, %%xmm12 \n\t" - "movaps %%xmm8, %%xmm13 \n\t" + "xorpd %%xmm12, %%xmm12 \n\t" + "xorpd %%xmm13, %%xmm13 \n\t" "prefetcht2 3 * 8(%%r10,%%rdi) \n\t" // prefetch c + 3*cs_c - "movaps %%xmm8, %%xmm14 \n\t" - "movaps %%xmm8, %%xmm15 \n\t" + "xorpd %%xmm14, %%xmm14 \n\t" + "xorpd %%xmm15, %%xmm15 \n\t" " \n\t" " \n\t" " \n\t" diff --git a/test/3m4m/Makefile b/test/3m4m/Makefile index bfe55935e5..66eb8dec42 100644 --- a/test/3m4m/Makefile +++ b/test/3m4m/Makefile @@ -45,8 +45,8 @@ # .PHONY: all \ - blis openblas mkl \ - blis-mt openblas-mt mkl-mt \ + blis-gemm-st openblas-gemm-st mkl-gemm-st acml-gemm-st \ + blis-gemm-mt openblas-gemm-mt mkl-gemm-mt acml-gemm-mt \ clean cleanx @@ -221,9 +221,6 @@ STR_NAT := -DSTR=\"asm\" STR_OBL := -DSTR=\"openblas\" STR_MKL := -DSTR=\"mkl\" STR_ACML := -DSTR=\"acml\" -STR_AMK := -DSTR=\"amk\" -STR_AKN := -DSTR=\"akn\" -STR_AMN := -DSTR=\"amn\" # Single or multithreaded string STR_ST := -DTHR_STR=\"st\" @@ -234,13 +231,9 @@ PDEF_ST := -DP_BEGIN=40 \ -DP_END=2000 \ -DP_INC=40 -PDEF_MT := -DP_BEGIN=400 \ - -DP_END=8000 \ - -DP_INC=400 - -ADJ_MK := -DADJ_MK -ADJ_KN := -DADJ_KN -ADJ_MN := -DADJ_MN +PDEF_MT := -DP_BEGIN=80 \ + -DP_END=4000 \ + -DP_INC=80 @@ -254,15 +247,19 @@ intel: blis-all openblas-all mkl-all amd: blis-all openblas-all acml-all -blis-all: blis blis-mt +blis-all: blis-gemm-st \ + blis-gemm-mt -openblas-all: openblas openblas-mt +openblas-all: openblas-gemm-st \ + openblas-gemm-mt -mkl-all: mkl mkl-mt +mkl-all: mkl-gemm-st \ + mkl-gemm-mt -acml-all: acml acml-mt +acml-all: acml-gemm-st \ + acml-gemm-mt -blis: \ +blis-gemm-st: \ test_sgemm_asm_blis_st.x \ test_dgemm_asm_blis_st.x \ \ @@ -281,16 +278,9 @@ blis: \ test_cgemm_4m1a_blis_st.x \ test_zgemm_4m1a_blis_st.x \ test_cgemm_asm_blis_st.x \ - test_zgemm_asm_blis_st.x \ - \ - test_cgemm_amk_blis_st.x \ - test_zgemm_amk_blis_st.x \ - test_cgemm_akn_blis_st.x \ - test_zgemm_akn_blis_st.x \ - test_cgemm_amn_blis_st.x \ - test_zgemm_amn_blis_st.x - -blis-mt: \ + test_zgemm_asm_blis_st.x + +blis-gemm-mt: \ test_sgemm_asm_blis_mt.x \ test_dgemm_asm_blis_mt.x \ \ @@ -311,27 +301,33 @@ blis-mt: \ test_cgemm_asm_blis_mt.x \ test_zgemm_asm_blis_mt.x -openblas: \ +openblas-gemm-st: \ + test_sgemm_openblas_st.x \ + test_dgemm_openblas_st.x \ test_cgemm_openblas_st.x \ test_zgemm_openblas_st.x -openblas-mt: \ +openblas-gemm-mt: \ test_cgemm_openblas_mt.x \ test_zgemm_openblas_mt.x -mkl: \ +mkl-gemm-st: \ + test_sgemm_mkl_st.x \ + test_dgemm_mkl_st.x \ test_cgemm_mkl_st.x \ test_zgemm_mkl_st.x -mkl-mt: \ +mkl-gemm-mt: \ test_cgemm_mkl_mt.x \ test_zgemm_mkl_mt.x -acml: \ +acml-gemm-st: \ + test_sgemm_acml_st.x \ + test_dgemm_acml_st.x \ test_cgemm_acml_st.x \ test_zgemm_acml_st.x -acml-mt: \ +acml-gemm-mt: \ test_cgemm_acml_mt.x \ test_zgemm_acml_mt.x @@ -458,26 +454,13 @@ test_z%_asm_blis_mt.o: test_%.c test_c%_asm_blis_mt.o: test_%.c $(CC) $(CFLAGS) $(PDEF_MT) $(DT_C) $(BLI_DEF) $(DNAT) $(STR_NAT) $(STR_MT) -c $< -o $@ -# blis asm (adjusted) -test_z%_amk_blis_st.o: test_%.c - $(CC) $(CFLAGS) $(PDEF_ST) $(DT_Z) $(BLI_DEF) $(DNAT) $(STR_AMK) $(STR_ST) $(ADJ_MK) -c $< -o $@ - -test_c%_amk_blis_st.o: test_%.c - $(CC) $(CFLAGS) $(PDEF_ST) $(DT_C) $(BLI_DEF) $(DNAT) $(STR_AMK) $(STR_ST) $(ADJ_MK) -c $< -o $@ - -test_z%_akn_blis_st.o: test_%.c - $(CC) $(CFLAGS) $(PDEF_ST) $(DT_Z) $(BLI_DEF) $(DNAT) $(STR_AKN) $(STR_ST) $(ADJ_KN) -c $< -o $@ - -test_c%_akn_blis_st.o: test_%.c - $(CC) $(CFLAGS) $(PDEF_ST) $(DT_C) $(BLI_DEF) $(DNAT) $(STR_AKN) $(STR_ST) $(ADJ_KN) -c $< -o $@ - -test_z%_amn_blis_st.o: test_%.c - $(CC) $(CFLAGS) $(PDEF_ST) $(DT_Z) $(BLI_DEF) $(DNAT) $(STR_AMN) $(STR_ST) $(ADJ_MN) -c $< -o $@ +# openblas +test_d%_openblas_st.o: test_%.c + $(CC) $(CFLAGS) $(PDEF_ST) $(DT_D) $(BLA_DEF) $(DNAT) $(STR_OBL) $(STR_ST) -c $< -o $@ -test_c%_amn_blis_st.o: test_%.c - $(CC) $(CFLAGS) $(PDEF_ST) $(DT_C) $(BLI_DEF) $(DNAT) $(STR_AMN) $(STR_ST) $(ADJ_MN) -c $< -o $@ +test_s%_openblas_st.o: test_%.c + $(CC) $(CFLAGS) $(PDEF_ST) $(DT_S) $(BLA_DEF) $(DNAT) $(STR_OBL) $(STR_ST) -c $< -o $@ -# openblas test_z%_openblas_st.o: test_%.c $(CC) $(CFLAGS) $(PDEF_ST) $(DT_Z) $(BLA_DEF) $(DNAT) $(STR_OBL) $(STR_ST) -c $< -o $@ @@ -491,6 +474,12 @@ test_c%_openblas_mt.o: test_%.c $(CC) $(CFLAGS) $(PDEF_MT) $(DT_C) $(BLA_DEF) $(DNAT) $(STR_OBL) $(STR_MT) -c $< -o $@ # mkl +test_d%_mkl_st.o: test_%.c + $(CC) $(CFLAGS) $(PDEF_ST) $(DT_D) $(BLA_DEF) $(DNAT) $(STR_MKL) $(STR_ST) -c $< -o $@ + +test_s%_mkl_st.o: test_%.c + $(CC) $(CFLAGS) $(PDEF_ST) $(DT_S) $(BLA_DEF) $(DNAT) $(STR_MKL) $(STR_ST) -c $< -o $@ + test_z%_mkl_st.o: test_%.c $(CC) $(CFLAGS) $(PDEF_ST) $(DT_Z) $(BLA_DEF) $(DNAT) $(STR_MKL) $(STR_ST) -c $< -o $@ @@ -504,6 +493,12 @@ test_c%_mkl_mt.o: test_%.c $(CC) $(CFLAGS) $(PDEF_MT) $(DT_C) $(BLA_DEF) $(DNAT) $(STR_MKL) $(STR_MT) -c $< -o $@ # acml +test_d%_acml_st.o: test_%.c + $(CC) $(CFLAGS) $(PDEF_ST) $(DT_D) $(BLA_DEF) $(DNAT) $(STR_ACML) $(STR_ST) -c $< -o $@ + +test_s%_acml_st.o: test_%.c + $(CC) $(CFLAGS) $(PDEF_ST) $(DT_S) $(BLA_DEF) $(DNAT) $(STR_ACML) $(STR_ST) -c $< -o $@ + test_z%_acml_st.o: test_%.c $(CC) $(CFLAGS) $(PDEF_ST) $(DT_Z) $(BLA_DEF) $(DNAT) $(STR_ACML) $(STR_ST) -c $< -o $@ diff --git a/test/3m4m/runme.sh b/test/3m4m/runme.sh index 2333b03678..bb65a5db56 100755 --- a/test/3m4m/runme.sh +++ b/test/3m4m/runme.sh @@ -4,16 +4,24 @@ exec_root="test" out_root="output" -sys="stampede" +sys="blis" +#sys="stampede" #sys="wahlberg" # Bind threads to processors. #export OMP_PROC_BIND=true #export GOMP_CPU_AFFINITY="0 2 4 6 8 10 12 14 1 3 5 7 9 11 13 15" -export GOMP_CPU_AFFINITY="0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15" +#export GOMP_CPU_AFFINITY="0 1 2 3 4 5 6 7" +export GOMP_CPU_AFFINITY="0 1 2 3 4 5 6 7" +#export GOMP_CPU_AFFINITY="0 2 4 6 1 3 5 7" +#export GOMP_CPU_AFFINITY="0 4 1 5 2 6 3 7" # Modify LD_LIBRARY_PATH. -if [ ${sys} = "stampede" ]; then +if [ ${sys} = "blis" ]; then + + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH" + +elif [ ${sys} = "stampede" ]; then # A hack to use libiomp5 with gcc. export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/apps/intel/13/composer_xe_2013.2.146/compiler/lib/intel64" @@ -25,7 +33,15 @@ elif [ ${sys} = "wahlberg" ]; then fi # Threading scheme to use when multithreading -if [ ${sys} = "stampede" ]; then +if [ ${sys} = "blis" ]; then + + jc_nt=1 # 5th loop + ic_nt=4 # 3rd loop + jr_nt=1 # 2nd loop + ir_nt=1 # 1st loop + nt=4 + +elif [ ${sys} = "stampede" ]; then jc_nt=2 # 5th loop ic_nt=8 # 3rd loop @@ -43,8 +59,8 @@ elif [ ${sys} = "wahlberg" ]; then fi # Threadedness to test. -threads="st" # mt" -threads_r="st" # mt" +threads="st mt" # st mt" +threads_r="st mt" # mt" # Datatypes to test. dts="z c" @@ -56,7 +72,12 @@ test_ops="${l3_ops}" test_ops_r="${l3_ops}" # Complex domain implementations to test. -if [ ${sys} = "stampede" ]; then +if [ ${sys} = "blis" ]; then + + #test_impls="openblas mkl 3mhw_blis 3m3_blis 3m2_blis 3m1_blis 4mhw_blis 4m1b_blis 4m1a_blis" + test_impls="openblas 3mhw_blis 3m3_blis 3m2_blis 3m1_blis 4mhw_blis 4m1b_blis 4m1a_blis" + +elif [ ${sys} = "stampede" ]; then test_impls="openblas mkl asm_blis 3mhw_blis 3m3_blis 3m2_blis 3m1_blis 4mhw_blis 4m1b_blis 4m1a_blis" #test_impls="openblas mkl asm_blis" @@ -64,12 +85,12 @@ if [ ${sys} = "stampede" ]; then elif [ ${sys} = "wahlberg" ]; then test_impls="openblas acml asm_blis 3mhw_blis 3m3_blis 3m2_blis 3m1_blis 4mhw_blis 4m1b_blis 4m1a_blis" - #test_impls="openblas acml asm_blis" + test_impls="openblas acml asm_blis" fi -test_impls="asm_blis amk_blis akn_blis amn_blis" # Real domain implementations to test. -test_impls_r="asm_blis" +#test_impls_r="openblas mkl asm_blis" +test_impls_r="openblas asm_blis" # First perform real test cases. for th in ${threads_r}; do @@ -106,7 +127,6 @@ for th in ${threads_r}; do #export GOMP_CPU_AFFINITY="0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15" fi - # Construct the name of the test executable. exec_name="${exec_root}_${dt}${op}_${im}_${th}.x" diff --git a/test/3m4m/test_gemm.c b/test/3m4m/test_gemm.c index afc2209bf9..0a45266d92 100644 --- a/test/3m4m/test_gemm.c +++ b/test/3m4m/test_gemm.c @@ -49,7 +49,7 @@ int main( int argc, char** argv ) dim_t p; dim_t p_begin, p_end, p_inc; int m_input, n_input, k_input; - num_t dt, dt_real; + num_t dt; char dt_ch; int r, n_repeats; trans_t transa; @@ -61,7 +61,6 @@ int main( int argc, char** argv ) double dtime_save; double gflops; - extern blksz_t* gemm_kc; bli_init(); @@ -70,7 +69,6 @@ int main( int argc, char** argv ) n_repeats = 3; dt = DT; - dt_real = bli_datatype_proj_to_real( DT ); p_begin = P_BEGIN; p_end = P_END; @@ -80,6 +78,11 @@ int main( int argc, char** argv ) n_input = -1; k_input = -1; +#if 0 + extern blksz_t* gemm_kc; + + num_t dt_real = bli_datatype_proj_to_real( DT ); + // Extract the kc blocksize for the requested datatype and its // real analogue. dim_t kc = bli_blksz_get_def( dt, gemm_kc ); @@ -92,14 +95,6 @@ int main( int argc, char** argv ) else if ( IND == BLIS_3M1 ) k_input = kc_real / 3; else if ( IND == BLIS_4M1A ) k_input = kc_real / 2; else k_input = kc_real; - - // Adjust the relative dimensions, if requested. -#if (defined ADJ_MK) - m_input = -2; k_input = -2; n_input = -1; -#elif (defined ADJ_KN) - k_input = -2; n_input = -2; m_input = -1; -#elif (defined ADJ_MN) - m_input = -2; n_input = -2; k_input = -1; #endif // Choose the char corresponding to the requested datatype. @@ -131,6 +126,7 @@ int main( int argc, char** argv ) for ( p = p_begin; p <= p_end; p += p_inc ) { + if ( m_input < 0 ) m = p / ( dim_t )abs(m_input); else m = ( dim_t ) m_input; if ( n_input < 0 ) n = p / ( dim_t )abs(n_input); diff --git a/test/thread_ranges/Makefile b/test/thread_ranges/Makefile new file mode 100644 index 0000000000..b506f20abe --- /dev/null +++ b/test/thread_ranges/Makefile @@ -0,0 +1,203 @@ +#!/bin/bash +# +# BLIS +# An object-based framework for developing high-performance BLAS-like +# libraries. +# +# Copyright (C) 2014, The University of Texas at Austin +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# - Neither the name of The University of Texas at Austin nor the names +# of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# +# Makefile +# +# Field G. Van Zee +# +# Makefile for standalone BLIS test drivers. +# + +# +# --- Makefile PHONY target definitions ---------------------------------------- +# + +.PHONY: all \ + test-ranges \ + clean cleanx + + + +# +# --- Makefile initialization -------------------------------------------------- +# + +# Define the name of the configuration file. +CONFIG_MK_FILE := config.mk + +# Define the name of the file containing build and architecture-specific +# makefile definitions. +MAKE_DEFS_FILE := make_defs.mk + +# Locations of important files. +ROOT_PATH := ../.. +CONFIG_DIR := config + + + +# +# --- Include makefile configuration file -------------------------------------- +# + +# Construct the path to the makefile configuration file that was generated by +# the configure script. +CONFIG_MK_PATH := $(ROOT_PATH)/$(CONFIG_MK_FILE) + +# Include the configuration file. +-include $(CONFIG_MK_PATH) + +# Detect whether we actually got the configuration file. If we didn't, then +# it is likely that the user has not yet generated it (via configure). +ifeq ($(strip $(CONFIG_MK_INCLUDED)),yes) +CONFIG_MK_PRESENT := yes +else +CONFIG_MK_PRESENT := no +endif + +# Now we have access to CONFIG_NAME, which tells us which sub-directory of the +# config directory to use as our configuration. +CONFIG_PATH := $(ROOT_PATH)/$(CONFIG_DIR)/$(CONFIG_NAME) + + + +# +# --- Include makefile definitions file ---------------------------------------- +# + +# Construct the path to the makefile definitions file residing inside of +# the configuration sub-directory. +MAKE_DEFS_MK_PATH := $(CONFIG_PATH)/$(MAKE_DEFS_FILE) + +# Include the makefile definitions file. +-include $(MAKE_DEFS_MK_PATH) + +# Detect whether we actually got the make definitios file. If we didn't, then +# it is likely that the configuration is invalid (or incomplete). +ifeq ($(strip $(MAKE_DEFS_MK_INCLUDED)),yes) +MAKE_DEFS_MK_PRESENT := yes +else +MAKE_DEFS_MK_PRESENT := no +endif + + + +# +# --- BLAS and LAPACK implementations ------------------------------------------ +# + +# BLIS library and header path. This is simply wherever it was installed. +BLIS_LIB_PATH := $(INSTALL_PREFIX)/lib +BLIS_INC_PATH := $(INSTALL_PREFIX)/include/blis + +# BLIS library. +BLIS_LIB := $(BLIS_LIB_PATH)/libblis.a + + + +# +# --- General build definitions ------------------------------------------------ +# + +TEST_SRC_PATH := . +TEST_OBJ_PATH := . + +# Gather all local object files. +TEST_OBJS := $(patsubst $(TEST_SRC_PATH)/%.c, \ + $(TEST_OBJ_PATH)/%.o, \ + $(wildcard $(TEST_SRC_PATH)/*.c)) + +# Override CFLAGS from make_defs.mk here, if desired. +#CFLAGS := -g -O2 -march=native + +# Add installed and local header paths to CFLAGS +CFLAGS += -I$(BLIS_INC_PATH) -I$(TEST_SRC_PATH) #-I$(ACML_INC_PATH) + +LINKER := $(CC) +LDFLAGS := #-L/home/00146/field/gnu/gcc-4.8.2/lib64 +LDFLAGS += -lgfortran -lm -lpthread -fopenmp + + +# Datatype +DT_S := -DDT=BLIS_FLOAT +DT_D := -DDT=BLIS_DOUBLE +DT_C := -DDT=BLIS_SCOMPLEX +DT_Z := -DDT=BLIS_DCOMPLEX + +# Problem size specification +PDEF_MT := -DP_BEGIN=400 \ + -DP_END=8000 \ + -DP_INC=400 + + + +# +# --- Targets/rules ------------------------------------------------------------ +# + +all: test-ranges + +test-ranges: \ + test_ranges.x + + + +# --Object file rules -- + +$(TEST_OBJ_PATH)/%.o: $(TEST_SRC_PATH)/%.c + $(CC) $(CFLAGS) -c $< -o $@ + +# blis asm +test_%.o: test_%.c + $(CC) $(CFLAGS) $(PDEF_MT) $(DT_D) -c $< -o $@ + + +# -- Executable file rules -- + +# NOTE: For the BLAS test drivers, we place the BLAS libraries before BLIS +# on the link command line in case BLIS was configured with the BLAS +# compatibility layer. This prevents BLIS from inadvertently getting called +# for the BLAS routines we are trying to test with. + +test_ranges.x: test_ranges.o $(BLIS_LIB) + $(LINKER) $< $(BLIS_LIB) $(LDFLAGS) -o $@ + + +# -- Clean rules -- + +clean: cleanx + +cleanx: + - $(RM_F) *.o *.x + diff --git a/test/thread_ranges/test_ranges.c b/test/thread_ranges/test_ranges.c new file mode 100644 index 0000000000..b34a80cb08 --- /dev/null +++ b/test/thread_ranges/test_ranges.c @@ -0,0 +1,313 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name of The University of Texas nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include +#include "blis.h" + + +//#define PRINT + +int main( int argc, char** argv ) +{ + bli_init(); + +#if 0 + obj_t a, b, c; + obj_t aa, bb, cc; + dim_t m, n, k; + num_t dt; + uplo_t uploa, uplob, uploc; + + { + dt = BLIS_DOUBLE; + + m = 6; + k = 6; + n = 6; + + bli_obj_create( dt, m, k, 0, 0, &a ); + bli_obj_create( dt, k, n, 0, 0, &b ); + bli_obj_create( dt, m, n, 0, 0, &c ); + + uploa = BLIS_UPPER; + uploa = BLIS_LOWER; + bli_obj_set_struc( BLIS_TRIANGULAR, a ); + bli_obj_set_uplo( uploa, a ); + bli_obj_set_diag_offset( -2, a ); + + uplob = BLIS_UPPER; + uplob = BLIS_LOWER; + bli_obj_set_struc( BLIS_TRIANGULAR, b ); + bli_obj_set_uplo( uplob, b ); + bli_obj_set_diag_offset( -2, b ); + + uploc = BLIS_UPPER; + //uploc = BLIS_LOWER; + //uploc = BLIS_ZEROS; + //uploc = BLIS_DENSE; + bli_obj_set_struc( BLIS_HERMITIAN, c ); + //bli_obj_set_struc( BLIS_TRIANGULAR, c ); + bli_obj_set_uplo( uploc, c ); + bli_obj_set_diag_offset( 1, c ); + + bli_obj_alias_to( a, aa ); (void)aa; + bli_obj_alias_to( b, bb ); (void)bb; + bli_obj_alias_to( c, cc ); (void)cc; + + bli_randm( &a ); + bli_randm( &b ); + bli_randm( &c ); + //bli_mkherm( &a ); + //bli_mktrim( &a ); + + bli_prune_unref_mparts( &cc, BLIS_M, + &aa, BLIS_N ); + + bli_printm( "c orig", &c, "%4.1f", "" ); + bli_printm( "c alias", &cc, "%4.1f", "" ); + bli_printm( "a orig", &a, "%4.1f", "" ); + bli_printm( "a alias", &aa, "%4.1f", "" ); + //bli_obj_print( "a struct", &a ); + } +#endif + + dim_t p_begin, p_max, p_inc; + gint_t m_input, n_input; + char uploa_ch; + doff_t diagoffa; + dim_t bf; + dim_t n_way; + char part_dim_ch; + bool_t go_fwd; + char out_ch; + + obj_t a; + + thrinfo_t thrinfo; + dim_t m, n; + uplo_t uploa; + bool_t part_m_dim, part_n_dim; + bool_t go_bwd; + dim_t p; + num_t dt; + dim_t start, end; + + dim_t width; + siz_t area; + + gint_t t_begin, t_stop, t_inc; + dim_t t; + + if ( argc == 13 ) + { + sscanf( argv[1], "%lu", &p_begin ); + sscanf( argv[2], "%lu", &p_max ); + sscanf( argv[3], "%lu", &p_inc ); + sscanf( argv[4], "%ld", &m_input ); + sscanf( argv[5], "%ld", &n_input ); + sscanf( argv[6], "%c", &uploa_ch ); + sscanf( argv[7], "%ld", &diagoffa ); + sscanf( argv[8], "%lu", &bf ); + sscanf( argv[9], "%lu", &n_way ); + sscanf( argv[10], "%c", &part_dim_ch ); + sscanf( argv[11], "%lu", &go_fwd ); + sscanf( argv[12], "%c", &out_ch ); + } + else + { + printf( "\n" ); + printf( " %s\n", argv[0] ); + printf( "\n" ); + printf( " Simulate the dimension ranges assigned to threads when\n" ); + printf( " partitioning a matrix for parallelism in BLIS.\n" ); + printf( "\n" ); + printf( " Usage:\n" ); + printf( "\n" ); + printf( " %s p_beg p_max p_inc m n uplo doff bf n_way part_dim go_fwd out\n", argv[0] ); + printf( "\n" ); + printf( " p_beg: the first problem size p to test.\n" ); + printf( " p_max: the maximum problem size p to test.\n" ); + printf( " p_inc: the increase in problem size p between tests.\n" ); + printf( " m: the m dimension:\n" ); + printf( " n: the n dimension:\n" ); + printf( " if m,n = -1: bind m,n to problem size p.\n" ); + printf( " if m,n = 0: bind m,n to p_max.\n" ); + printf( " if m,n > 0: hold m,n = c constant for all p.\n" ); + printf( " uplo: the uplo field of the matrix being partitioned:\n" ); + printf( " 'l': lower-stored (BLIS_LOWER)\n" ); + printf( " 'u': upper-stored (BLIS_UPPER)\n" ); + printf( " 'd': densely-stored (BLIS_DENSE)\n" ); + printf( " doff: the diagonal offset of the matrix being partitioned.\n" ); + printf( " bf: the simulated blocking factor. all thread ranges must\n" ); + printf( " be a multiple of bf, except for the range that contains\n" ); + printf( " the edge case (if one exists). the blocking factor\n" ); + printf( " would typically correspond to a register blocksize.\n" ); + printf( " n_way: the number of ways of parallelism for which we are\n" ); + printf( " partitioning (i.e.: the number of threads, or thread\n" ); + printf( " groups).\n" ); + printf( " part_dim: the dimension to partition:\n" ); + printf( " 'm': partition the m dimension.\n" ); + printf( " 'n': partition the n dimension.\n" ); + printf( " go_fwd: the direction to partition:\n" ); + printf( " '1': forward, e.g. left-to-right (part_dim = 'm') or\n" ); + printf( " top-to-bottom (part_dim = 'n')\n" ); + printf( " '0': backward, e.g. right-to-left (part_dim = 'm') or\n" ); + printf( " bottom-to-top (part_dim = 'n')\n" ); + printf( " NOTE: reversing the direction does not change the\n" ); + printf( " subpartitions' widths, but it does change which end of\n" ); + printf( " the index range receives the edge case, if it exists.\n" ); + printf( " out: the type of output per thread-column:\n" ); + printf( " 'w': the width (and area) of the thread's subpartition\n" ); + printf( " 'r': the actual ranges of the thread's subpartition\n" ); + printf( " where the start and end points of each range are\n" ); + printf( " inclusive and exclusive, respectively.\n" ); + printf( "\n" ); + + exit(1); + } + + if ( m_input == 0 ) m_input = p_max; + if ( n_input == 0 ) n_input = p_max; + + if ( part_dim_ch == 'm' ) { part_m_dim = TRUE; part_n_dim = FALSE; } + else { part_m_dim = FALSE; part_n_dim = TRUE; } + + go_bwd = !go_fwd; + + if ( uploa_ch == 'l' ) uploa = BLIS_LOWER; + else if ( uploa_ch == 'u' ) uploa = BLIS_UPPER; + else uploa = BLIS_DENSE; + + if ( part_n_dim ) + { + if ( bli_is_upper( uploa ) ) { t_begin = n_way-1; t_stop = -1; t_inc = -1; } + else /* if lower or dense */ { t_begin = 0; t_stop = n_way; t_inc = 1; } + } + else // if ( part_m_dim ) + { + if ( bli_is_lower( uploa ) ) { t_begin = n_way-1; t_stop = -1; t_inc = -1; } + else /* if upper or dense */ { t_begin = 0; t_stop = n_way; t_inc = 1; } + } + + printf( "\n" ); + printf( " part: %3s doff: %3ld bf: %3ld output: %s\n", + ( part_n_dim ? ( go_fwd ? "l2r" : "r2l" ) + : ( go_fwd ? "t2b" : "b2t" ) ), + diagoffa, bf, + ( out_ch == 'w' ? "width(area)" : "ranges" ) ); + printf( " uplo: %3c nt: %3ld\n", uploa_ch, n_way ); + printf( "\n" ); + + printf( " " ); + for ( t = t_begin; t != t_stop; t += t_inc ) + { + if ( part_n_dim ) + { + if ( t == t_begin ) printf( "left... " ); + else if ( t == t_stop-t_inc ) printf( " ...right" ); + else printf( " " ); + } + else // if ( part_m_dim ) + { + if ( t == t_begin ) printf( "top... " ); + else if ( t == t_stop-t_inc ) printf( " ...bottom" ); + else printf( " " ); + } + } + printf( "\n" ); + + + printf( "%4c x %4c ", 'm', 'n' ); + for ( t = t_begin; t != t_stop; t += t_inc ) + { + printf( "%9s %lu ", "thread", t ); + } + printf( "\n" ); + printf( "-------------" ); + for ( t = t_begin; t != t_stop; t += t_inc ) + { + printf( "-------------" ); + } + printf( "\n" ); + + + for ( p = p_begin; p <= p_max; p += p_inc ) + { + if ( m_input < 0 ) m = ( dim_t )p; + else m = ( dim_t )m_input; + if ( n_input < 0 ) n = ( dim_t )p; + else n = ( dim_t )n_input; + + dt = BLIS_DOUBLE; + + bli_obj_create( dt, m, n, 0, 0, &a ); + + bli_obj_set_struc( BLIS_TRIANGULAR, a ); + bli_obj_set_uplo( uploa, a ); + bli_obj_set_diag_offset( diagoffa, a ); + + bli_randm( &a ); + + printf( "%4lu x %4lu ", m, n ); + + for ( t = t_begin; t != t_stop; t += t_inc ) + { + thrinfo.n_way = n_way; + thrinfo.work_id = t; + + if ( part_n_dim && go_fwd ) + area = bli_get_range_weighted_l2r( &thrinfo, &a, bf, &start, &end ); + else if ( part_n_dim && go_bwd ) + area = bli_get_range_weighted_r2l( &thrinfo, &a, bf, &start, &end ); + else if ( part_m_dim && go_fwd ) + area = bli_get_range_weighted_t2b( &thrinfo, &a, bf, &start, &end ); + else // ( part_m_dim && go_bwd ) + area = bli_get_range_weighted_b2t( &thrinfo, &a, bf, &start, &end ); + + width = end - start; + + if ( out_ch == 'w' ) printf( "%4lu(%6lu) ", width, area ); + else printf( "[%4lu,%4lu) ", start, end ); + } + + printf( "\n" ); + + bli_obj_free( &a ); + } + + bli_finalize(); + + return 0; +} + diff --git a/testsuite/src/test_addv.c b/testsuite/src/test_addv.c index b8909944a3..8f2cf880ec 100644 --- a/testsuite/src/test_addv.c +++ b/testsuite/src/test_addv.c @@ -164,7 +164,7 @@ void libblis_test_addv_experiment( test_params_t* params, } // Estimate the performance of the best experiment repeat. - *perf = ( 2.0 * m ) / time_min / FLOPS_PER_UNIT_PERF; + *perf = ( 1.0 * m ) / time_min / FLOPS_PER_UNIT_PERF; if ( bli_obj_is_complex( x ) ) *perf *= 2.0; // Perform checks. diff --git a/testsuite/src/test_gemm.c b/testsuite/src/test_gemm.c index 0d5732f524..43e619c579 100644 --- a/testsuite/src/test_gemm.c +++ b/testsuite/src/test_gemm.c @@ -174,12 +174,12 @@ void libblis_test_gemm_experiment( test_params_t* params, if ( bli_obj_is_real( c ) ) { bli_setsc( 1.2, 0.0, &alpha ); - bli_setsc( -1.0, 0.0, &beta ); + bli_setsc( 0.9, 0.0, &beta ); } else { bli_setsc( 1.2, 0.8, &alpha ); - bli_setsc( -1.0, 1.0, &beta ); + bli_setsc( 0.9, 1.0, &beta ); } // Randomize A, B, and C, and save C. diff --git a/testsuite/src/test_libblis.c b/testsuite/src/test_libblis.c index a493a3a853..55a98bd961 100644 --- a/testsuite/src/test_libblis.c +++ b/testsuite/src/test_libblis.c @@ -595,39 +595,27 @@ void libblis_test_output_params_struct( FILE* os, test_params_t* params ) libblis_test_fprintf_c( os, "\n" ); libblis_test_fprintf_c( os, "--- BLIS library info -------------------------------------\n" ); libblis_test_fprintf_c( os, "\n" ); - libblis_test_fprintf_c( os, "version string %s\n", bli_info_get_version_str() ); + libblis_test_fprintf_c( os, "version string %s\n", bli_info_get_version_str() ); libblis_test_fprintf_c( os, "\n" ); - libblis_test_fprintf_c( os, "--- BLIS config header ---\n" ); + libblis_test_fprintf_c( os, "--- BLIS configuration info ---\n" ); libblis_test_fprintf_c( os, "\n" ); - libblis_test_fprintf_c( os, "integer type size (bits) %d\n", ( int )int_type_size ); - libblis_test_fprintf_c( os, "# of floating-point types %d\n", ( int )bli_info_get_num_fp_types() ); - libblis_test_fprintf_c( os, "maximum type size %d\n", ( int )bli_info_get_max_type_size() ); + libblis_test_fprintf_c( os, "integer type size (bits) %d\n", ( int )int_type_size ); libblis_test_fprintf_c( os, "\n" ); - libblis_test_fprintf_c( os, "maximum number of threads %d\n", ( int )bli_info_get_max_num_threads() ); + libblis_test_fprintf_c( os, "SIMD alignment (bytes) %d\n", ( int )bli_info_get_simd_align_size() ); + libblis_test_fprintf_c( os, "Page size (bytes) %d\n", ( int )bli_info_get_page_size() ); libblis_test_fprintf_c( os, "\n" ); - libblis_test_fprintf_c( os, "SIMD alignment (bytes) %d\n", ( int )bli_info_get_simd_align_size() ); + libblis_test_fprintf_c( os, "memory alignment (bytes) \n" ); + libblis_test_fprintf_c( os, " stack address (def: simd) %d\n", ( int )bli_info_get_stack_buf_align_size() ); + libblis_test_fprintf_c( os, " obj_t address (def: simd) %d\n", ( int )bli_info_get_heap_addr_align_size() ); + libblis_test_fprintf_c( os, " obj_t stride (def: simd) %d\n", ( int )bli_info_get_heap_stride_align_size() ); + libblis_test_fprintf_c( os, " pool block addr (def: page) %d\n", ( int )bli_info_get_pool_addr_align_size() ); libblis_test_fprintf_c( os, "\n" ); - libblis_test_fprintf_c( os, "stack memory allocation \n" ); - libblis_test_fprintf_c( os, " address alignment (bytes) %d\n", ( int )bli_info_get_stack_buf_align_size() ); + libblis_test_fprintf_c( os, "BLAS compatibility layer \n" ); + libblis_test_fprintf_c( os, " enabled? %d\n", ( int )bli_info_get_enable_blas2blis() ); + libblis_test_fprintf_c( os, " integer type size (bits) %d\n", ( int )bli_info_get_blas2blis_int_type_size() ); libblis_test_fprintf_c( os, "\n" ); - libblis_test_fprintf_c( os, "dynamic memory allocation \n" ); - libblis_test_fprintf_c( os, " address alignment %d\n", ( int )bli_info_get_heap_addr_align_size() ); - libblis_test_fprintf_c( os, " stride alignment %d\n", ( int )bli_info_get_heap_stride_align_size() ); - libblis_test_fprintf_c( os, "\n" ); - libblis_test_fprintf_c( os, "contiguous memory allocation \n" ); - libblis_test_fprintf_c( os, " # of mc x kc blocks %d\n", ( int )bli_info_get_num_mc_x_kc_blocks() ); - libblis_test_fprintf_c( os, " # of kc x nc blocks %d\n", ( int )bli_info_get_num_kc_x_nc_blocks() ); - libblis_test_fprintf_c( os, " # of mc x nc blocks %d\n", ( int )bli_info_get_num_mc_x_nc_blocks() ); - libblis_test_fprintf_c( os, " block address alignment %d\n", ( int )bli_info_get_contig_addr_align_size() ); - libblis_test_fprintf_c( os, " max preload byte offset %d\n", ( int )bli_info_get_max_preload_byte_offset() ); - libblis_test_fprintf_c( os, " actual pool sizes (bytes) \n" ); - libblis_test_fprintf_c( os, " for mc x kc blocks of A %d\n", ( int )bli_info_get_mk_pool_size() ); - libblis_test_fprintf_c( os, " for kc x nc panels of B %d\n", ( int )bli_info_get_kn_pool_size() ); - libblis_test_fprintf_c( os, " for mc x nc panels of C %d\n", ( int )bli_info_get_mn_pool_size() ); - libblis_test_fprintf_c( os, "\n" ); - libblis_test_fprintf_c( os, "BLAS compatibility layer \n" ); - libblis_test_fprintf_c( os, " enabled? %d\n", ( int )bli_info_get_enable_blas2blis() ); - libblis_test_fprintf_c( os, " integer type size (bits) %d\n", ( int )bli_info_get_blas2blis_int_type_size() ); + libblis_test_fprintf_c( os, "CBLAS compatibility layer \n" ); + libblis_test_fprintf_c( os, " enabled? %d\n", ( int )bli_info_get_enable_cblas() ); libblis_test_fprintf_c( os, "\n" ); libblis_test_fprintf_c( os, "floating-point types s d c z \n" ); libblis_test_fprintf_c( os, " sizes (bytes) %7u %7u %7u %7u\n", sizeof(float), @@ -862,18 +850,6 @@ void libblis_test_output_params_struct( FILE* os, test_params_t* params ) libblis_test_fprintf_c( os, "\n" ); libblis_test_fprintf_c( os, "--- BLIS misc. other info ---\n" ); libblis_test_fprintf_c( os, "\n" ); - libblis_test_fprintf_c( os, "micro-panel alignment (bytes) s d c z \n" ); - libblis_test_fprintf_c( os, " A (left matrix) %7d %7d %7d %7d\n", - ( int )bli_info_get_upanel_a_align_size_s(), - ( int )bli_info_get_upanel_a_align_size_d(), - ( int )bli_info_get_upanel_a_align_size_c(), - ( int )bli_info_get_upanel_a_align_size_z() ); - libblis_test_fprintf_c( os, " B (right matrix) %7d %7d %7d %7d\n", - ( int )bli_info_get_upanel_b_align_size_s(), - ( int )bli_info_get_upanel_b_align_size_d(), - ( int )bli_info_get_upanel_b_align_size_c(), - ( int )bli_info_get_upanel_b_align_size_z() ); - libblis_test_fprintf_c( os, "\n" ); libblis_test_fprintf_c( os, "level-2 cache blocksizes s d c z \n" ); libblis_test_fprintf_c( os, " m dimension %7d %7d %7d %7d\n", ( int )bli_info_get_default_l2_mc_s(), @@ -1468,7 +1444,7 @@ void libblis_test_op_driver( test_params_t* params, dt_char = params->datatype_char[dt]; // Build a commented column label string. - libblis_test_build_col_labels_string( op, label_str ); + libblis_test_build_col_labels_string( params, op, label_str ); // Output the column label string. libblis_test_fprintf( stdout, "%s\n", label_str ); @@ -1731,7 +1707,7 @@ void libblis_test_build_dims_string( test_op_t* op, // % dtoper_params_storage m n k gflops resid result -void libblis_test_build_col_labels_string( test_op_t* op, char* l_str ) +void libblis_test_build_col_labels_string( test_params_t* params, test_op_t* op, char* l_str ) { unsigned int n_spaces; char blank_str[64]; @@ -1751,7 +1727,9 @@ void libblis_test_build_col_labels_string( test_op_t* op, char* l_str ) "
_ " ); } - n_spaces = 1; + if ( params->output_matlab_format ) n_spaces = 11; + else n_spaces = 1; + fill_string_with_n_spaces( blank_str, n_spaces ); sprintf( &l_str[strlen(l_str)], "%s", blank_str ); diff --git a/testsuite/src/test_libblis.h b/testsuite/src/test_libblis.h index 6a747d4649..8bdbd37be3 100644 --- a/testsuite/src/test_libblis.h +++ b/testsuite/src/test_libblis.h @@ -362,7 +362,7 @@ void libblis_test_build_filename_string( char* prefix_str, char* op_str, char* funcname_str ); -void libblis_test_build_col_labels_string( test_op_t* op, char* l_str ); +void libblis_test_build_col_labels_string( test_params_t* params, test_op_t* op, char* l_str ); void fill_string_with_n_spaces( char* str, unsigned int n_spaces ); diff --git a/testsuite/src/test_scal2m.c b/testsuite/src/test_scal2m.c index 2816c8a08e..8cd1fdd2bc 100644 --- a/testsuite/src/test_scal2m.c +++ b/testsuite/src/test_scal2m.c @@ -181,7 +181,7 @@ void libblis_test_scal2m_experiment( test_params_t* params, } // Estimate the performance of the best experiment repeat. - *perf = ( 2.0 * m * n ) / time_min / FLOPS_PER_UNIT_PERF; + *perf = ( 1.0 * m * n ) / time_min / FLOPS_PER_UNIT_PERF; if ( bli_obj_is_complex( y ) ) *perf *= 4.0; // Perform checks. diff --git a/testsuite/src/test_scal2v.c b/testsuite/src/test_scal2v.c index 184b50067a..803e5e2cf8 100644 --- a/testsuite/src/test_scal2v.c +++ b/testsuite/src/test_scal2v.c @@ -179,7 +179,7 @@ void libblis_test_scal2v_experiment( test_params_t* params, } // Estimate the performance of the best experiment repeat. - *perf = ( 2.0 * m ) / time_min / FLOPS_PER_UNIT_PERF; + *perf = ( 1.0 * m ) / time_min / FLOPS_PER_UNIT_PERF; if ( bli_obj_is_complex( y ) ) *perf *= 4.0; // Perform checks. diff --git a/testsuite/src/test_subv.c b/testsuite/src/test_subv.c index 2b7f8c3cea..c6379bf1eb 100644 --- a/testsuite/src/test_subv.c +++ b/testsuite/src/test_subv.c @@ -165,7 +165,7 @@ void libblis_test_subv_experiment( test_params_t* params, } // Estimate the performance of the best experiment repeat. - *perf = ( 2.0 * m ) / time_min / FLOPS_PER_UNIT_PERF; + *perf = ( 1.0 * m ) / time_min / FLOPS_PER_UNIT_PERF; if ( bli_obj_is_complex( x ) ) *perf *= 2.0; // Perform checks. diff --git a/version b/version index c946ee6160..699c6c6d4e 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.1.6 +0.1.8