Cutlass
CUDA Templates for Linear Algebra Subroutines and Solvers
wmma_matrix.h
Go to the documentation of this file.
1 /***************************************************************************************************
2  * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without modification, are permitted
5  * provided that the following conditions are met:
6  * * Redistributions of source code must retain the above copyright notice, this list of
7  * conditions and the following disclaimer.
8  * * Redistributions in binary form must reproduce the above copyright notice, this list of
9  * conditions and the following disclaimer in the documentation and/or other materials
10  * provided with the distribution.
11  * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
12  * to endorse or promote products derived from this software without specific prior written
13  * permission.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
17  * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
19  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
20  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
21  * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
22  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23  *
24  **************************************************************************************************/
28 #pragma once
29 
30 #if defined(__CUDACC__) && (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 700)
31 
32 // Dependent header files should use the following macro to guard all code using
33 // nvcuda::wmma:: to enable compilation for CUDA Compute Capabilities < sm_70.
34 // Earlier shader models not support Tensor Cores.
35 #define CUTLASS_USE_WMMA_API
36 
37 #include "stdio.h"
38 
39 #include <crt/mma.h>
40 #include <cutlass/fragment.h>
41 #include <cutlass/load_store.h>
42 #include <cutlass/matrix_traits.h>
43 #include <cutlass/shape.h>
44 #include <cutlass/vector.h>
45 
46 namespace cutlass {
47 
49 
51 template <MatrixLayout::Kind kLayout_>
52 struct WmmaLayout {
53  typedef nvcuda::wmma::col_major Layout;
54 };
55 
57 template <>
58 struct WmmaLayout<MatrixLayout::kRowMajor> {
59  typedef nvcuda::wmma::row_major Layout;
60 };
61 
63 
65 template <GemmOperand::Kind kOperand_,
66  MatrixLayout::Kind kLayout_,
67  typename Scalar_,
68  typename WmmaShape_>
69 struct WmmaMatrix {};
70 
72 
74 template <MatrixLayout::Kind kLayout_, typename Scalar_, typename WmmaShape_>
75 struct WmmaMatrix<GemmOperand::kA, kLayout_, Scalar_, WmmaShape_>
76  : public nvcuda::wmma::fragment<
78  nvcuda::wmma::matrix_a,
80  WmmaShape_::kW,
81  WmmaShape_::kH,
82  WmmaShape_::kD,
84  Scalar_,
86  typename WmmaLayout<kLayout_>::Layout> {
88  typedef WmmaMatrix<GemmOperand::kA, kLayout_, Scalar_, WmmaShape_> This_;
89 
91  CUTLASS_DEVICE This_& operator=(Scalar_ const& x) {
92  nvcuda::wmma::fill_fragment(*this, x);
93  return *this;
94  }
95 
97  CUTLASS_DEVICE void load(Scalar_ const* pointer, int const stride) {
98  nvcuda::wmma::load_matrix_sync(*this, pointer, stride);
99  }
100 
102  CUTLASS_DEVICE void store(Scalar_* pointer, int const stride) const {
103  nvcuda::wmma::store_matrix_sync(pointer, *this, stride);
104  }
105 };
106 
108 
110 template <MatrixLayout::Kind kLayout_, typename Scalar_, typename WmmaShape_>
111 struct WmmaMatrix<GemmOperand::kB, kLayout_, Scalar_, WmmaShape_>
112  : public nvcuda::wmma::fragment<
114  nvcuda::wmma::matrix_b,
116  WmmaShape_::kW,
117  WmmaShape_::kH,
118  WmmaShape_::kD,
120  Scalar_,
122  typename WmmaLayout<kLayout_>::Layout> {
124  typedef WmmaMatrix<GemmOperand::kB, kLayout_, Scalar_, WmmaShape_> This_;
125 
127  CUTLASS_DEVICE This_& operator=(Scalar_ const& x) {
128  nvcuda::wmma::fill_fragment(*this, x);
129  return *this;
130  }
131 
133  CUTLASS_DEVICE void load(Scalar_ const* pointer, int const stride) {
134  nvcuda::wmma::load_matrix_sync(*this, pointer, stride);
135  }
136 
138  CUTLASS_DEVICE void store(Scalar_* pointer, int const stride) const {
139  nvcuda::wmma::store_matrix_sync(pointer, *this, stride);
140  }
141 };
142 
144 
146 template <MatrixLayout::Kind kLayout_, typename Scalar_, typename WmmaShape_>
147 struct WmmaMatrix<GemmOperand::kC, kLayout_, Scalar_, WmmaShape_>
148  : public nvcuda::wmma::fragment<
150  nvcuda::wmma::accumulator,
152  WmmaShape_::kW,
153  WmmaShape_::kH,
154  WmmaShape_::kD,
156  Scalar_> {
158  typedef WmmaMatrix<GemmOperand::kC, kLayout_, Scalar_, WmmaShape_> This_;
160  static MatrixLayout::Kind const kLayout = kLayout_;
161 
163  CUTLASS_DEVICE This_& operator=(Scalar_ const& x) {
164  nvcuda::wmma::fill_fragment(*this, x);
165  return *this;
166  }
167 
169  CUTLASS_DEVICE void load(Scalar_ const* pointer, int const stride) {
170  bool const kIsRowMajor = kLayout == MatrixLayout::kRowMajor;
171  nvcuda::wmma::load_matrix_sync(
172  *this,
173  pointer,
174  stride,
175  kIsRowMajor ? nvcuda::wmma::mem_row_major : nvcuda::wmma::mem_col_major);
176  }
177 
179  CUTLASS_DEVICE void store(Scalar_* pointer, int const stride) const {
180  bool const kIsRowMajor = kLayout == MatrixLayout::kRowMajor;
181  nvcuda::wmma::store_matrix_sync(
182  pointer,
183  *this,
184  stride,
185  kIsRowMajor ? nvcuda::wmma::mem_row_major : nvcuda::wmma::mem_col_major);
186  }
187 };
188 
190 
191 } // namespace cutlass
192 
193 #endif // defined CUTLASS_USE_WMMA_API
Definition: convert.h:33
Definition: matrix_traits.h:36
Defines abstractions for efficiently loading and storing vectors to memory.
Defines a 1D vector of elements held in the registers of each thread.
Kind
Definition: matrix_traits.h:36
Kind
Definition: matrix_traits.h:43
Defines Shape implementing the Layout concept for representing a 4D hypercube of objects.
Defines properties of matrices used to denote layout and operands to GEMM kernels.
Defines Fragment, a statically-sized array for storing parts of matrices within a thread&#39;s registers...