gemm_fragment_stream.h
Go to the documentation of this file.
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#pragma once
#include <cutlass/fragment_stream.h>
#include <cutlass/matrix_traits.h>
#include <cutlass/gemm/gemm_operand.h>
#include <cutlass/gemm/gemm_traits.h>
namespace cutlass {
namespace gemm {
template <GemmOperand::Kind Usage, typename Scalar_, MatrixLayout::Kind Layout,
typename ThreadBlockTile_, int Threads, int ScalarsPerInst, typename Index_ = int,
typename DestinationSkew_ = Shape<0, 0, 0, 0> >
struct GemmFragmentStreamTraits {
static GemmOperand::Kind const kUsage = Usage;
typedef Scalar_ Scalar;
static MatrixLayout::Kind const kLayout = Layout;
typedef ThreadBlockTile_ ThreadBlockTile;
static int const kThreads = Threads;
static int const kAccessSize = ScalarsPerInst;
typedef Index_ Index;
typedef typename ShapeDiv<DestinationSkew_, Shape<ScalarsPerInst, ScalarsPerInst, ScalarsPerInst,
1> >::Shape DestinationSkew;
typedef GemmMultiplicandTraits<ThreadBlockTile, kUsage, kLayout> MultiplicandTraits;
typedef typename MultiplicandTraits::Shape ScalarTile;
typedef typename ReshapeTile<ScalarTile, kAccessSize>::Tile VectorizedTile;
typedef TileTraitsDefault<VectorizedTile, kThreads> TileTraits;
typedef FragmentStream<
TileTraits,
TileLoadIterator<TileTraits, Scalar,
MultiplicandTraits::kKstrided ? IteratorAdvance::kH : IteratorAdvance::kW,
MemorySpace::kGlobal, Index>,
TileStoreIterator<TileTraits, Scalar,
MultiplicandTraits::kKstrided ? IteratorAdvance::kH : IteratorAdvance::kW,
MemorySpace::kShared, Index, Scalar, IteratorFragment::kScalar,
DestinationSkew> >
FragmentStream;
};
template <typename Traits_>
struct GemmFragmentStream
: public FragmentStream<
typename Traits_::TileTraits,
TileLoadIterator<typename Traits_::TileTraits, typename Traits_::Scalar,
Traits_::MultiplicandTraits::kKstrided ? IteratorAdvance::kH
: IteratorAdvance::kW,
MemorySpace::kGlobal, typename Traits_::Index>,
TileStoreIterator<typename Traits_::TileTraits, typename Traits_::Scalar,
Traits_::MultiplicandTraits::kKstrided ? IteratorAdvance::kH
: IteratorAdvance::kW,
MemorySpace::kShared, typename Traits_::Index, typename Traits_::Scalar,
IteratorFragment::kScalar, typename Traits_::DestinationSkew> > {
typedef Traits_ Traits;
typedef typename Traits::FragmentStream Base;
//
// FragmentStream concept
//
typedef typename Traits::Scalar Scalar;
typedef typename Base::LoadIterator LoadIterator;
typedef typename Base::StoreIterator StoreIterator;
typedef typename Base::Convert Convert;
typedef typename Base::Fragment Fragment;
typedef typename Base::StoreFragment StoreFragment;
typedef typename Base::Storage Storage;
// Parameters type
// typedef typename Base::Params BaseParams;
typedef typename Traits::Index Index;
//
// Nested class definitions
//
typedef typename Traits::Scalar const *Pointer;
struct Params : public Base::Params {
//
// Methods
//
template <typename GemmDesc_>
CUTLASS_HOST_DEVICE int initialize(GemmDesc_ const &desc,
typename Traits::Scalar const *pointer, Index ldm) {
return this->load_params.initialize(pointer, ldm * Traits::MultiplicandTraits::Shape::kH, ldm,
Traits::kAccessSize);
}
};
//
// Static member functions
//
static CUTLASS_DEVICE void shared_store_fence() { Base::shared_store_fence(); }
//
// Methods
//
CUTLASS_DEVICE
GemmFragmentStream() {}
CUTLASS_DEVICE
GemmFragmentStream(Params const &params, Coord<3> const &bounds,
Coord<3> const &block_offset = make_Coord(0, 0, 0))
: Base(params, ProjectOperand<Traits::kUsage, Traits::MultiplicandTraits::kKstrided>::project(
bounds) +
make_Coord(1, 0, 0),
ProjectOperand<Traits::kUsage, Traits::MultiplicandTraits::kKstrided>::project(
block_offset)) {}
CUTLASS_DEVICE
void load() { Base::load(); }
CUTLASS_DEVICE
void commit() { Base::commit(); }
CUTLASS_DEVICE
void residue(Coord<3> const &bounds, Coord<3> const &block_offset) {
this->initialize_predicates(bounds, block_offset);
this->fetch.clear();
}
CUTLASS_DEVICE
void initialize_predicates(Coord<3> const &bounds, Coord<3> const &block_offset) {
Base::initialize_predicates(
ProjectOperand<Traits::kUsage, Traits::MultiplicandTraits::kKstrided>::project(bounds) +
make_Coord(1, 0, 0),
ProjectOperand<Traits::kUsage, Traits::MultiplicandTraits::kKstrided>::project(
block_offset));
}
};
}
}
cutlass::gemm::GemmMultiplicandTraits::Shape
nv_std::conditional< kKstrided, Shape< 1, ThreadBlockTile::kD, GetExtent< Usage, ThreadBlockTile >::kExtent >, Shape< 1, GetExtent< Usage, ThreadBlockTile >::kExtent, ThreadBlockTile::kD > >::type Shape
Map the ThreadBlockShape onto (kH, kW) dimensions for A and B operand.
Definition: gemm_operand.h:86
cutlass::gemm::GemmMultiplicandTraits::kKstrided
static bool const kKstrided
Definition: gemm_operand.h:81
cutlass::gemm::GemmFragmentStreamTraits::Scalar
Scalar_ Scalar
Scalar data type.
Definition: gemm_fragment_stream.h:50
cutlass::gemm::GemmFragmentStreamTraits::MultiplicandTraits
GemmMultiplicandTraits< ThreadBlockTile, kUsage, kLayout > MultiplicandTraits
Traits of multiplicand.
Definition: gemm_fragment_stream.h:72
cutlass::gemm::GemmFragmentStreamTraits::kAccessSize
static int const kAccessSize
Scalars per instruction.
Definition: gemm_fragment_stream.h:62
cutlass::MemorySpace::kShared
Definition: load_store.h:42
cutlass
Definition: convert.h:34
cutlass::gemm::GemmFragmentStream::StoreIterator
Base::StoreIterator StoreIterator
Defines the store iterator.
Definition: gemm_fragment_stream.h:127
cutlass::gemm::GemmFragmentStream::Params::initialize
CUTLASS_HOST_DEVICE int initialize(GemmDesc_ const &desc, typename Traits::Scalar const *pointer, Index ldm)
Initializes parameters.
Definition: gemm_fragment_stream.h:162
gemm_traits.h
Defines structural properties of complete GEMM computation.
cutlass::gemm::GemmFragmentStream::Base
Traits::FragmentStream Base
Base class.
Definition: gemm_fragment_stream.h:114
fragment_stream.h
An abstraction for implementing a stream loading a tile and storing a tile using a pair of tile itera...
cutlass::MemorySpace::kGlobal
Definition: load_store.h:43
cutlass::make_Coord
CUTLASS_HOST_DEVICE Coord< 1 > make_Coord(int _0)
Helper to make a 2-element coordinate.
Definition: coord.h:241
cutlass::gemm::GemmFragmentStreamTraits::VectorizedTile
ReshapeTile< ScalarTile, kAccessSize >::Tile VectorizedTile
Reshape for vectorized access.
Definition: gemm_fragment_stream.h:78
cutlass::gemm::GemmFragmentStream::Index
Traits::Index Index
Index type.
Definition: gemm_fragment_stream.h:145
cutlass::gemm::GemmFragmentStreamTraits::FragmentStream
FragmentStream< TileTraits, TileLoadIterator< TileTraits, Scalar, MultiplicandTraits::kKstrided ? IteratorAdvance::kH :IteratorAdvance::kW, MemorySpace::kGlobal, Index >, TileStoreIterator< TileTraits, Scalar, MultiplicandTraits::kKstrided ? IteratorAdvance::kH :IteratorAdvance::kW, MemorySpace::kShared, Index, Scalar, IteratorFragment::kScalar, DestinationSkew > > FragmentStream
Define the tile stream.
Definition: gemm_fragment_stream.h:93
cutlass::gemm::GemmFragmentStream::Traits
Traits_ Traits
Traits.
Definition: gemm_fragment_stream.h:111
cutlass::FragmentStream< Traits_::TileTraits, TileLoadIterator< Traits_::TileTraits, Traits_::Scalar, Traits_::MultiplicandTraits::kKstrided ? IteratorAdvance::kH :IteratorAdvance::kW, MemorySpace::kGlobal, Traits_::Index >, TileStoreIterator< Traits_::TileTraits, Traits_::Scalar, Traits_::MultiplicandTraits::kKstrided ? IteratorAdvance::kH :IteratorAdvance::kW, MemorySpace::kShared, Traits_::Index, Traits_::Scalar, IteratorFragment::kScalar, Traits_::DestinationSkew > >::fetch
Fragment fetch
Fragment fetched by load iterator.
Definition: fragment_stream.h:135
cutlass::IteratorAdvance::kH
Definition: tile_iterator.h:97
cutlass::gemm::GemmFragmentStream::GemmFragmentStream
CUTLASS_DEVICE GemmFragmentStream()
Definition: gemm_fragment_stream.h:181
cutlass::gemm::GemmFragmentStreamTraits::TileTraits
TileTraitsDefault< VectorizedTile, kThreads > TileTraits
Define structure of stripmined tile.
Definition: gemm_fragment_stream.h:81
cutlass::gemm::GemmFragmentStreamTraits::ScalarTile
MultiplicandTraits::Shape ScalarTile
Scalar tile shape.
Definition: gemm_fragment_stream.h:75
cutlass::gemm::GemmFragmentStream::shared_store_fence
static CUTLASS_DEVICE void shared_store_fence()
The memory fence for shared stores.
Definition: gemm_fragment_stream.h:174
cutlass::gemm::GemmFragmentStreamTraits
Defines a FragmentStream by mapping GEMM dimensions onto contiguous and strided dimensions.
Definition: gemm_fragment_stream.h:45
cutlass::gemm::GemmFragmentStream::GemmFragmentStream
CUTLASS_DEVICE GemmFragmentStream(Params const &params, Coord< 3 > const &bounds, Coord< 3 > const &block_offset=make_Coord(0, 0, 0))
Constructor - bounds and block offset are aligned to GEMM coordinates (K, N, M)
Definition: gemm_fragment_stream.h:185
cutlass::gemm::GemmFragmentStream::Fragment
Base::Fragment Fragment
Loaded fragment type.
Definition: gemm_fragment_stream.h:133
cutlass::gemm::GemmFragmentStream
GEMM Fragment Stream.
Definition: gemm_fragment_stream.h:98
cutlass::gemm::GemmFragmentStream::Pointer
Traits::Scalar const * Pointer
The pointer.
Definition: gemm_fragment_stream.h:152
cutlass::TileLoadIterator
An iterator implementing Tile Load Iterator Concept for loading a tile from memory.
Definition: tile_iterator.h:308
cutlass::gemm::GemmFragmentStream::commit
CUTLASS_DEVICE void commit()
Commits the fragment.
Definition: gemm_fragment_stream.h:199
cutlass::gemm::GemmFragmentStream::Storage
Base::Storage Storage
Destination storage.
Definition: gemm_fragment_stream.h:139
gemm_operand.h
Defines constant expressions for mapping GEMM problem size and strides onto pitch-linear memory...
cutlass::FragmentStream
Manages a pair of iterators to stream data from global memory to shared.
Definition: fragment_stream.h:50
cutlass::gemm::GemmMultiplicandTraits
Definition: gemm_operand.h:66
cutlass::gemm::GemmFragmentStreamTraits::kLayout
static MatrixLayout::Kind const kLayout
Layout of the operand.
Definition: gemm_fragment_stream.h:53
cutlass::gemm::GemmFragmentStream::Scalar
Traits::Scalar Scalar
Scalar type.
Definition: gemm_fragment_stream.h:121
cutlass::gemm::GemmFragmentStream::Params
Parameters object.
Definition: gemm_fragment_stream.h:155
CUTLASS_HOST_DEVICE
#define CUTLASS_HOST_DEVICE
Definition: cutlass.h:37
cutlass::gemm::GemmFragmentStreamTraits::Index
Index_ Index
Index type.
Definition: gemm_fragment_stream.h:65
cutlass::ShapeDiv
Definition: shape.h:124
cutlass::Shape
A Shape implementing Layout Concept describing the dimensions of a cube.
Definition: shape.h:63
cutlass::gemm::GemmFragmentStream::Convert
Base::Convert Convert
Converts between tiles.
Definition: gemm_fragment_stream.h:130
cutlass::gemm::ProjectOperand
Definition: gemm_operand.h:94
cutlass::Coord
Statically-sized array specifying Coords within a tensor.
Definition: coord.h:48
cutlass::gemm::GemmFragmentStreamTraits::kThreads
static int const kThreads
Number of threads.
Definition: gemm_fragment_stream.h:59
cutlass::MatrixLayout::Kind
Kind
Definition: matrix_traits.h:36
cutlass::gemm::GemmFragmentStream::StoreFragment
Base::StoreFragment StoreFragment
Stored fragment type.
Definition: gemm_fragment_stream.h:136
cutlass::gemm::GemmFragmentStream::LoadIterator
Base::LoadIterator LoadIterator
Defines the load iterator.
Definition: gemm_fragment_stream.h:124
cutlass::gemm::GemmFragmentStreamTraits::kUsage
static GemmOperand::Kind const kUsage
Indicates identity of multiplicand.
Definition: gemm_fragment_stream.h:47
cutlass::ReshapeTile::Tile
Tile_ Tile
Definition: tile.h:43
cutlass::IteratorAdvance::kW
Definition: tile_iterator.h:97
cutlass::gemm::GemmFragmentStreamTraits::DestinationSkew
ShapeDiv< DestinationSkew_, Shape< ScalarsPerInst, ScalarsPerInst, ScalarsPerInst, 1 > >::Shape DestinationSkew
Skew added to shared memory tile.
Definition: gemm_fragment_stream.h:69
cutlass::GemmOperand::Kind
Kind
Definition: matrix_traits.h:43
cutlass::gemm::GemmFragmentStream::residue
CUTLASS_DEVICE void residue(Coord< 3 > const &bounds, Coord< 3 > const &block_offset)
TODO - Recomputes predicates and clears fetch registers.
Definition: gemm_fragment_stream.h:203
cutlass::gemm::GemmFragmentStreamTraits::ThreadBlockTile
ThreadBlockTile_ ThreadBlockTile
Shape of the thread block tile (K, N, M)
Definition: gemm_fragment_stream.h:56
matrix_traits.h
Defines properties of matrices used to denote layout and operands to GEMM kernels.
cutlass::gemm::GemmFragmentStream::initialize_predicates
CUTLASS_DEVICE void initialize_predicates(Coord< 3 > const &bounds, Coord< 3 > const &block_offset)
Recomputes predicates aligned to GEMM coordinates (K, N, M)
Definition: gemm_fragment_stream.h:211
cutlass::IteratorFragment::kScalar
Definition: tile_iterator.h:102
cutlass::gemm::GemmFragmentStream::load
CUTLASS_DEVICE void load()
Loads the fragment.
Definition: gemm_fragment_stream.h:195
cutlass::TileStoreIterator
An iterator implementing Tile Store Iterator Concept for storing a tile to memory.
Definition: tile_iterator.h:556