igemm_epilogue.h
Go to the documentation of this file.
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#pragma once
#include <cutlass/convert.h>
#include <cutlass/fragment.h>
#include <cutlass/gemm/gemm_global_stream.h>
#include <cutlass/gemm/gemm_shared_stream.h>
#include <cutlass/gemm/igemm_global_tile.h>
#include <cutlass/reshape_tile.h>
#include <cutlass/tile_iterator.h>
namespace cutlass {
namespace gemm {
template <int kElements_>
struct IgemmFloatToInt8Converter {
typedef Fragment<float, kElements_> InputFragment;
typedef Fragment<int8_t, kElements_> OutputFragment;
// We are packing 4 floats into int32 registers so we need kElements to be multiple of 4.
static_assert(kElements_ % 4 == 0, "kElements must be multiple of 4");
CUTLASS_DEVICE IgemmFloatToInt8Converter() {}
CUTLASS_DEVICE void transform(InputFragment const& src, OutputFragment& dst) {
transform(src, 0, dst);
}
template <typename Fragment_>
CUTLASS_DEVICE void transform(Fragment_ const& src, int offset, OutputFragment& dst) {
// The inputs.
float4 const* src_f4 = reinterpret_cast<float4 const*>(&src[0]);
// The outputs.
int* dst_int = reinterpret_cast<int*>(&dst[0]);
// Iterate over the floats and pack them together to produce ints.
for (int i = 0; i < kElements_ / 4; ++i) {
// Read the float4.
float4 f4 = src_f4[i];
// Clamp the 4 elements of the floats to the [-128, +127] range.
float x = fmaxf(-128.f, fminf(127.f, f4.x));
float y = fmaxf(-128.f, fminf(127.f, f4.y));
float z = fmaxf(-128.f, fminf(127.f, f4.z));
float w = fmaxf(-128.f, fminf(127.f, f4.w));
// Convert to integers.
int ix = (int)x;
int iy = (int)y;
int iz = (int)z;
int iw = (int)w;
// Extract the lower bytes to build an int32 with 4 int8.
asm volatile("prmt.b32 %0, %0, %1, 0x1140;" : "+r"(ix) : "r"(iy));
asm volatile("prmt.b32 %0, %0, %1, 0x1140;" : "+r"(iz) : "r"(iw));
asm volatile("prmt.b32 %0, %0, %1, 0x5410;" : "+r"(ix) : "r"(iz));
// Store the int.
dst_int[i] = ix;
}
}
};
template <typename InputScalar_, typename OutputFragment_>
struct IgemmGlobalStoreTransformer {
typedef Convert<Fragment<InputScalar_, OutputFragment_::kElements>, OutputFragment_> Transformer;
};
template <int kElements_>
struct IgemmGlobalStoreTransformer<float, Fragment<int8_t, kElements_> > {
typedef IgemmFloatToInt8Converter<kElements_> Transformer;
};
template <int kElements_>
struct IgemmInt8ToFloatConverter {
typedef Fragment<int8_t, kElements_> InputFragment;
typedef Fragment<float, kElements_> OutputFragment;
// We are unpacking 4 int8s from int32.
static_assert(kElements_ % 4 == 0, "kElements must be multiple of 4");
CUTLASS_DEVICE IgemmInt8ToFloatConverter() {}
CUTLASS_DEVICE void transform(InputFragment const& src, OutputFragment& dst) {
transform(src, 0, dst);
}
template <typename Fragment_>
CUTLASS_DEVICE void transform(Fragment_ const& src, int offset, OutputFragment& dst) {
// The inputs.
int const* src_int = reinterpret_cast<int const*>(&src[0]);
// The outputs.
float4* dst_f4 = reinterpret_cast<float4*>(&dst[0]);
// Iterate over the int8 and unpack them together to produce floats.
for (int i = 0; i < kElements_ / 4; ++i) {
// Read the int.
int ix, iy, iz, iw = src_int[i];
// Extract the 4 bytes.
asm volatile("prmt.b32 %0, 0x0, %1, 0x4440;" : "=r"(ix) : "r"(iw));
asm volatile("prmt.b32 %0, 0x0, %1, 0x4441;" : "=r"(iy) : "r"(iw));
asm volatile("prmt.b32 %0, 0x0, %1, 0x4442;" : "=r"(iz) : "r"(iw));
asm volatile("prmt.b32 %0, 0x0, %1, 0x4443;" : "=r"(iw) : "r"(iw));
// The floats.
float fx, fy, fz, fw;
// Convert to floats (make sure we generate I2F.F32.S8).
asm volatile("cvt.rn.f32.s8 %0, %1;" : "=f"(fx) : "r"(ix));
asm volatile("cvt.rn.f32.s8 %0, %1;" : "=f"(fy) : "r"(iy));
asm volatile("cvt.rn.f32.s8 %0, %1;" : "=f"(fz) : "r"(iz));
asm volatile("cvt.rn.f32.s8 %0, %1;" : "=f"(fw) : "r"(iw));
// Store the float4.
dst_f4[i] = make_float4(fx, fy, fz, fw);
}
}
};
template <typename InputFragment_, typename OutputScalar_>
struct IgemmGlobalLoadTransformer {
typedef Convert<InputFragment_, Fragment<OutputScalar_, InputFragment_::kElements> > Transformer;
};
template <int kElements_>
struct IgemmGlobalLoadTransformer<Fragment<int8_t, kElements_>, float> {
typedef IgemmInt8ToFloatConverter<kElements_> Transformer;
};
template <typename InputScalar_, typename OutputFragment_>
struct IgemmSharedStoreTransformer {
typedef Convert<Fragment<InputScalar_, OutputFragment_::kElements>, OutputFragment_> Transformer;
};
template <typename IgemmConfig_, typename EpilogueFunctor_, typename Index_>
struct IgemmEpilogueTraitsHelper
: public GemmEpilogueTraitsHelper<IgemmConfig_, EpilogueFunctor_, Index_> {
typedef GemmEpilogueTraitsHelper<IgemmConfig_, EpilogueFunctor_, Index_> Base;
typedef IgemmConfig_ IgemmConfig;
typedef typename Base::Scalar Scalar;
typedef typename Base::Iterations Iterations;
typedef typename Base::Delta Delta;
typedef typename Base::GlobalLoadTileTraits GlobalLoadTileTraits;
typedef GemmGlobalIteratorCd<GlobalLoadTileTraits> GlobalLoadIteratorC;
typedef typename GlobalLoadIteratorC::Fragment GlobalFragmentC;
typedef
typename IgemmGlobalLoadTransformer<GlobalFragmentC, Scalar>::Transformer GlobalTransformerC;
typedef typename Base::GlobalStoreTileTraits GlobalStoreTileTraits;
typedef GemmGlobalIteratorCd<GlobalStoreTileTraits> GlobalStoreIteratorD;
typedef typename GlobalStoreIteratorD::Fragment GlobalFragmentD;
typedef
typename IgemmGlobalStoreTransformer<Scalar, GlobalFragmentD>::Transformer GlobalTransformerD;
typedef typename Base::SharedStoreTileTraits SharedStoreTileTraits;
typedef TileStoreIterator<SharedStoreTileTraits,
typename SharedStoreTileTraits::Scalar,
IteratorAdvance::kH,
MemorySpace::kGlobal>
SharedStoreIteratorD;
typedef typename SharedStoreIteratorD::Fragment SharedStoreFragmentD;
typedef typename IgemmSharedStoreTransformer<typename IgemmConfig::Accumulators::Element,
SharedStoreFragmentD>::Transformer
SharedStoreTransformerD;
typedef typename Base::SharedLoadTileTraits SharedLoadTileTraits;
typedef TileLoadIterator<SharedLoadTileTraits,
typename SharedLoadTileTraits::Scalar,
IteratorAdvance::kH,
MemorySpace::kShared>
SharedLoadIteratorD;
};
template <
typename IgemmConfig_,
typename EpilogueFunctor_,
typename Index_ = int,
typename Helper_ = IgemmEpilogueTraitsHelper<IgemmConfig_, EpilogueFunctor_, Index_> >
struct IgemmEpilogueTraits : public GemmEpilogueTraits<
// The output tile.
typename IgemmConfig_::OutputTile,
// The accumulators.
typename IgemmConfig_::Accumulators,
// The global iterator for C.
typename Helper_::GlobalLoadIteratorC,
// The transformer for C.
typename Helper_::GlobalTransformerC,
// The transformer for D.
typename Helper_::GlobalTransformerD,
// The global iterator for D.
typename Helper_::GlobalStoreIteratorD,
// The iterator to store D to shared memory.
typename Helper_::SharedStoreIteratorD,
// The shared store transformer for D.
typename Helper_::SharedStoreTransformerD,
// The iterator to load D from shared memory.
typename Helper_::SharedLoadIteratorD,
// The iterations.
typename Helper_::Iterations,
// The strides between iterations.
typename Helper_::Delta,
// The functor to be used in the epilogue.
EpilogueFunctor_,
// The index.
Index_> {
static bool const kInt8Output =
platform::is_same<typename IgemmConfig_::ScalarC, int8_t>::value != 0;
};
template <typename GemmEpilogueTraits_, bool = GemmEpilogueTraits_::kInt8Output>
struct IgemmEpilogue : public GemmEpilogue<GemmEpilogueTraits_> {
typedef GemmEpilogue<GemmEpilogueTraits_> Base;
CUTLASS_DEVICE IgemmEpilogue(typename Base::Params const& params_,
typename Base::SharedStorage& shared_storage_,
typename Base::Index m_,
typename Base::Index n_)
: Base(params_, shared_storage_, m_, n_) {}
};
template <typename GemmEpilogueTraits_>
struct IgemmEpilogue<GemmEpilogueTraits_, true> : public GemmEpilogue<GemmEpilogueTraits_> {
typedef GemmEpilogue<GemmEpilogueTraits_> Base;
CUTLASS_DEVICE IgemmEpilogue(typename Base::Params const& params_,
typename Base::SharedStorage& shared_storage_,
typename Base::Index m_,
typename Base::Index n_)
: Base(params_, shared_storage_, m_, n_) {}
};
} // namespace gemm
} // namespace cutlass
cutlass::gemm::GemmGlobalTileCdTraits
Definition: gemm_global_tile.h:116
cutlass::gemm::IgemmEpilogueTraits
Definition: igemm_epilogue.h:255
cutlass::MemorySpace::kShared
Definition: load_store.h:42
cutlass::gemm::IgemmEpilogueTraitsHelper::Delta
Base::Delta Delta
The iterations strides.
Definition: igemm_epilogue.h:198
cutlass::TileStoreIterator::Fragment
Base::Fragment Fragment
Fragment definition.
Definition: tile_iterator.h:682
cutlass::gemm::IgemmEpilogueTraitsHelper::SharedStoreTileTraits
Base::SharedStoreTileTraits SharedStoreTileTraits
The traits class for the shared iterator to store D to shared memory.
Definition: igemm_epilogue.h:221
cutlass::gemm::IgemmEpilogueTraitsHelper::GlobalTransformerD
IgemmGlobalStoreTransformer< Scalar, GlobalFragmentD >::Transformer GlobalTransformerD
The transformer from accumulators to shared memory fragments.
Definition: igemm_epilogue.h:218
cutlass
Definition: convert.h:33
cutlass::gemm::IgemmEpilogueTraitsHelper::SharedLoadTileTraits
Base::SharedLoadTileTraits SharedLoadTileTraits
The traits class for the shared iterator to load D from shared memory.
Definition: igemm_epilogue.h:235
cutlass::gemm::IgemmEpilogueTraitsHelper::SharedLoadIteratorD
TileLoadIterator< SharedLoadTileTraits, typename SharedLoadTileTraits::Scalar, IteratorAdvance::kH, MemorySpace::kShared > SharedLoadIteratorD
The shared iterator to load D from shared memory.
Definition: igemm_epilogue.h:241
cutlass::gemm::GemmEpilogueTraitsHelper
Definition: gemm_epilogue_traits.h:171
cutlass::gemm::IgemmEpilogue::Base
GemmEpilogue< GemmEpilogueTraits_ > Base
The base class.
Definition: igemm_epilogue.h:292
cutlass::gemm::GemmEpilogue::Params
Traits::Params Params
The params.
Definition: gemm_epilogue.h:57
cutlass::gemm::GemmEpilogue
Definition: gemm_epilogue.h:53
cutlass::gemm::IgemmGlobalLoadTransformer
Definition: igemm_epilogue.h:167
cutlass::platform::is_same
std::is_same (false specialization)
Definition: platform.h:412
tile_iterator.h
Defines the Tile Traits concept and iterators for loading and storing to tiles efficiently.
cutlass::gemm::IgemmInt8ToFloatConverter::IgemmInt8ToFloatConverter
CUTLASS_DEVICE IgemmInt8ToFloatConverter()
Ctor.
Definition: igemm_epilogue.h:123
cutlass::gemm::IgemmEpilogueTraitsHelper::SharedStoreFragmentD
SharedStoreIteratorD::Fragment SharedStoreFragmentD
The fragment that needs to be passed to that store iterator.
Definition: igemm_epilogue.h:229
cutlass::gemm::GemmEpilogueTraitsHelper< IgemmConfig_, EpilogueFunctor_, Index_ >::Scalar
EpilogueFunctor_::Scalar Scalar
The scalar.
Definition: gemm_epilogue_traits.h:173
cutlass::gemm::IgemmEpilogueTraitsHelper
Definition: igemm_epilogue.h:186
cutlass::MemorySpace::kGlobal
Definition: load_store.h:43
cutlass::gemm::IgemmInt8ToFloatConverter::InputFragment
Fragment< int8_t, kElements_ > InputFragment
The input fragment.
Definition: igemm_epilogue.h:115
cutlass::gemm::IgemmEpilogue
Definition: igemm_epilogue.h:290
cutlass::gemm::IgemmFloatToInt8Converter
Definition: igemm_epilogue.h:45
cutlass::gemm::IgemmFloatToInt8Converter::transform
CUTLASS_DEVICE void transform(Fragment_ const &src, int offset, OutputFragment &dst)
Transform a fragment.
Definition: igemm_epilogue.h:64
cutlass::gemm::GemmEpilogue::SharedStorage
Traits::SharedStorage SharedStorage
The shared storage.
Definition: gemm_epilogue.h:59
cutlass::Fragment
A template defining Fragment Concept.
Definition: fragment.h:99
cutlass::IteratorAdvance::kH
Definition: tile_iterator.h:62
cutlass::gemm::IgemmInt8ToFloatConverter::transform
CUTLASS_DEVICE void transform(InputFragment const &src, OutputFragment &dst)
Transform a fragment.
Definition: igemm_epilogue.h:126
cutlass::gemm::IgemmEpilogueTraitsHelper::Scalar
Base::Scalar Scalar
The scalar type of the epilogue.
Definition: igemm_epilogue.h:194
cutlass::gemm::IgemmEpilogueTraitsHelper::GlobalFragmentC
GlobalLoadIteratorC::Fragment GlobalFragmentC
The fragment that needs to be produced by the load iterator.
Definition: igemm_epilogue.h:205
cutlass::gemm::IgemmFloatToInt8Converter::transform
CUTLASS_DEVICE void transform(InputFragment const &src, OutputFragment &dst)
Transform a fragment.
Definition: igemm_epilogue.h:58
cutlass::gemm::IgemmFloatToInt8Converter::OutputFragment
Fragment< int8_t, kElements_ > OutputFragment
The output fragment.
Definition: igemm_epilogue.h:49
cutlass::gemm::IgemmEpilogueTraitsHelper::GlobalStoreIteratorD
GemmGlobalIteratorCd< GlobalStoreTileTraits > GlobalStoreIteratorD
The iterator to store to shared memory.
Definition: igemm_epilogue.h:213
cutlass::gemm::IgemmEpilogueTraitsHelper::SharedStoreTransformerD
IgemmSharedStoreTransformer< typename IgemmConfig::Accumulators::Element, SharedStoreFragmentD >::Transformer SharedStoreTransformerD
The transformer from accumulators to shared memory fragments.
Definition: igemm_epilogue.h:233
cutlass::gemm::IgemmEpilogueTraits::kInt8Output
static bool const kInt8Output
Do we output in int8?
Definition: igemm_epilogue.h:283
cutlass::TileLoadIterator
An iterator implementing Tile Load Iterator Concept for loading a tile from memory.
Definition: tile_iterator.h:302
cutlass::gemm::IgemmSharedStoreTransformer::Transformer
Convert< Fragment< InputScalar_, OutputFragment_::kElements >, OutputFragment_ > Transformer
Definition: igemm_epilogue.h:180
cutlass::gemm::IgemmEpilogue< GemmEpilogueTraits_, true >::Base
GemmEpilogue< GemmEpilogueTraits_ > Base
The base class.
Definition: igemm_epilogue.h:307
reshape_tile.h
Defines a type for restructuring a tile.
cutlass::gemm::IgemmEpilogueTraitsHelper::GlobalLoadTileTraits
Base::GlobalLoadTileTraits GlobalLoadTileTraits
The traits class for the iterator.
Definition: igemm_epilogue.h:201
cutlass::gemm::IgemmInt8ToFloatConverter::OutputFragment
Fragment< float, kElements_ > OutputFragment
The output fragment.
Definition: igemm_epilogue.h:117
cutlass::gemm::IgemmEpilogueTraitsHelper::Base
GemmEpilogueTraitsHelper< IgemmConfig_, EpilogueFunctor_, Index_ > Base
The base class.
Definition: igemm_epilogue.h:189
cutlass::gemm::IgemmEpilogue::IgemmEpilogue
CUTLASS_DEVICE IgemmEpilogue(typename Base::Params const &params_, typename Base::SharedStorage &shared_storage_, typename Base::Index m_, typename Base::Index n_)
Ctor.
Definition: igemm_epilogue.h:295
cutlass::gemm::GemmSharedLoadTileDTraits
Definition: gemm_shared_tile.h:335
cutlass::gemm::GemmEpilogue::Index
Traits::Index Index
The index.
Definition: gemm_epilogue.h:93
cutlass::gemm::IgemmEpilogueTraitsHelper::GlobalFragmentD
GlobalStoreIteratorD::Fragment GlobalFragmentD
The fragment that needs to be passed to that store iterator.
Definition: igemm_epilogue.h:215
cutlass::gemm::IgemmEpilogueTraitsHelper::GlobalLoadIteratorC
GemmGlobalIteratorCd< GlobalLoadTileTraits > GlobalLoadIteratorC
The iterator to store to shared memory.
Definition: igemm_epilogue.h:203
static_assert
#define static_assert(__e, __m)
Definition: platform.h:145
cutlass::gemm::IgemmEpilogueTraitsHelper::IgemmConfig
IgemmConfig_ IgemmConfig
The config.
Definition: igemm_epilogue.h:191
cutlass::gemm::IgemmEpilogue< GemmEpilogueTraits_, true >::IgemmEpilogue
CUTLASS_DEVICE IgemmEpilogue(typename Base::Params const &params_, typename Base::SharedStorage &shared_storage_, typename Base::Index m_, typename Base::Index n_)
Ctor.
Definition: igemm_epilogue.h:310
cutlass::Shape
A Shape implementing Layout Concept describing the dimensions of a cube.
Definition: shape.h:64
cutlass::gemm::IgemmFloatToInt8Converter::IgemmFloatToInt8Converter
CUTLASS_DEVICE IgemmFloatToInt8Converter()
Ctor.
Definition: igemm_epilogue.h:55
cutlass::Fragment::Element
Element_ Element
The element.
Definition: fragment.h:108
cutlass::gemm::IgemmFloatToInt8Converter::InputFragment
Fragment< float, kElements_ > InputFragment
The input fragment.
Definition: igemm_epilogue.h:47
cutlass::gemm::GemmEpilogueTraits
Definition: gemm_epilogue_traits.h:70
cutlass::gemm::GemmGlobalIteratorCd
Definition: gemm_global_tile.h:348
cutlass::gemm::IgemmSharedStoreTransformer
Definition: igemm_epilogue.h:179
gemm_global_stream.h
Implements efficient loading of the thread block-level tile from global memory and storing to shared ...
cutlass::TileIteratorBase< TileTraits_, TileTraits_::Scalar, IteratorAdvance::kH, MemorySpace::kGlobal, Index_ >::Fragment
Fragment< FragmentElement, ShapeCount< Iterations >::kCount *kAccessSize > Fragment
The fragment.
Definition: tile_iterator.h:154
cutlass::Convert
Definition: convert.h:38
cutlass::gemm::IgemmGlobalStoreTransformer< float, Fragment< int8_t, kElements_ > >::Transformer
IgemmFloatToInt8Converter< kElements_ > Transformer
Definition: igemm_epilogue.h:107
cutlass::gemm::IgemmEpilogueTraitsHelper::Iterations
Base::Iterations Iterations
The iterations.
Definition: igemm_epilogue.h:196
cutlass::gemm::IgemmEpilogueTraitsHelper::GlobalTransformerC
IgemmGlobalLoadTransformer< GlobalFragmentC, Scalar >::Transformer GlobalTransformerC
The transformer from loaded data to math fragment.
Definition: igemm_epilogue.h:208
cutlass::gemm::IgemmEpilogueTraitsHelper::GlobalStoreTileTraits
Base::GlobalStoreTileTraits GlobalStoreTileTraits
The traits class for the iterator.
Definition: igemm_epilogue.h:211
cutlass::gemm::IgemmGlobalLoadTransformer::Transformer
Convert< InputFragment_, Fragment< OutputScalar_, InputFragment_::kElements > > Transformer
Definition: igemm_epilogue.h:168
fragment.h
Defines Fragment, a statically-sized array for storing parts of matrices within a thread&#39;s registers...
cutlass::gemm::GemmSharedStoreTileDTraits::Scalar
platform::remove_const< Scalar_ >::type Scalar
The scalar.
Definition: gemm_shared_tile.h:266
cutlass::gemm::IgemmInt8ToFloatConverter::transform
CUTLASS_DEVICE void transform(Fragment_ const &src, int offset, OutputFragment &dst)
Transform a fragment.
Definition: igemm_epilogue.h:132
cutlass::gemm::IgemmGlobalStoreTransformer::Transformer
Convert< Fragment< InputScalar_, OutputFragment_::kElements >, OutputFragment_ > Transformer
Definition: igemm_epilogue.h:102
gemm_shared_stream.h
Defines abstractions for managing loading and storing fragments to shared memory in the efficient GEM...
cutlass::gemm::IgemmGlobalStoreTransformer
Definition: igemm_epilogue.h:101
cutlass::gemm::IgemmEpilogueTraitsHelper::SharedStoreIteratorD
TileStoreIterator< SharedStoreTileTraits, typename SharedStoreTileTraits::Scalar, IteratorAdvance::kH, MemorySpace::kGlobal > SharedStoreIteratorD
The shared iterator to store D to shared memory.
Definition: igemm_epilogue.h:227
cutlass::gemm::IgemmGlobalLoadTransformer< Fragment< int8_t, kElements_ >, float >::Transformer
IgemmInt8ToFloatConverter< kElements_ > Transformer
Definition: igemm_epilogue.h:173
convert.h
Defines conversion operations among Fragments of different base type.
cutlass::gemm::IgemmInt8ToFloatConverter
Definition: igemm_epilogue.h:113
cutlass::gemm::GemmSharedLoadTileDTraits::Scalar
platform::remove_const< Scalar_ >::type Scalar
The scalar.
Definition: gemm_shared_tile.h:337
igemm_global_tile.h
Implements tile iterators to partition the thread block tile into 2D subtiles and efficiently load ea...
cutlass::gemm::GemmSharedStoreTileDTraits
Definition: gemm_shared_tile.h:264
cutlass::TileStoreIterator
An iterator implementing Tile Store Iterator Concept for storing a tile to memory.
Definition: tile_iterator.h:620