tile_iterator.h
Go to the documentation of this file.
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#pragma once
#include <cutlass/fragment.h>
#include <cutlass/load_store.h>
#include <cutlass/predicate_vector.h>
#include <cutlass/vector.h>
namespace cutlass {
struct IteratorAdvance {
enum Kind { kD, kH, kW };
};
struct IteratorFragment {
enum Kind { kScalar, kWmmaMatrix };
};
template <typename Tile_, typename Delta_, typename Iterations_, typename ThreadOffset_>
struct TileTraits {
typedef Tile_ Tile;
typedef Delta_ Delta;
typedef Iterations_ Iterations;
typedef ThreadOffset_ ThreadOffset;
};
template <typename Traits_,
typename Scalar_,
IteratorAdvance::Kind Advance_ = IteratorAdvance::kH,
MemorySpace::Kind MemorySpace = MemorySpace::kGeneric,
typename Index_ = int,
typename FragmentElement_ = Scalar_,
IteratorFragment::Kind IteratorFragment_ = IteratorFragment::kScalar,
typename Skew_ = Shape<0, 0, 0, 0> >
struct TileIteratorBase {
typedef Traits_ Traits;
typedef Scalar_ Scalar;
typedef FragmentElement_ FragmentElement;
static IteratorAdvance::Kind const kAdvance = Advance_;
static IteratorFragment::Kind const kIteratorFragment = IteratorFragment_;
static MemorySpace::Kind const kMemorySpace = MemorySpace;
typedef Index_ Index;
typedef Skew_ Skew;
typedef typename Traits::Tile Tile;
typedef typename Traits::Delta Delta;
typedef typename Traits::ImmediateOffsetStrides ImmediateOffsetStrides;
typedef typename Traits::Iterations Iterations;
typedef typename Traits::ThreadOffset ThreadOffset;
static int const kAccessSize = Tile::kC;
typedef typename Vectorize<FragmentElement, kAccessSize>::Type AccessType;
static int const kFragmentSize =
(kIteratorFragment == IteratorFragment::kWmmaMatrix ? 16 : sizeof(AccessType));
typedef Fragment<Scalar, ShapeCount<Tile>::kCount, kFragmentSize> Storage;
typedef Fragment<FragmentElement, ShapeCount<Iterations>::kCount * kAccessSize> Fragment;
typedef FragmentIterator<Fragment, Iterations, AccessType> FragmentIterator;
typedef FragmentConstIterator<Fragment, Iterations, AccessType> FragmentConstIterator;
typedef typename FragmentIterator::FragmentShape FragmentShape;
typedef PredicateVector<ShapeCount<Iterations>::kCount> PredicateVector;
//
// Params struct
//
struct Params {
Index stride_d;
Index stride_h;
Index stride_w;
Index inc_d;
Index inc_h;
Index inc_w;
Index inc_advance;
CUTLASS_HOST_DEVICE
int initialize(Index _stride_d,
Index _stride_h,
Index _stride_w,
Index _inc_d,
Index _inc_h,
Index _inc_w,
Index _inc_advance) {
stride_d = _stride_d;
stride_h = _stride_h;
stride_w = _stride_w;
inc_d = _inc_d;
inc_h = _inc_h;
inc_w = _inc_w;
inc_advance = _inc_advance;
return 0;
}
CUTLASS_HOST_DEVICE
int initialize(Index _stride_d, Index _stride_h, Index _stride_w) {
stride_d = _stride_d;
stride_h = _stride_h;
stride_w = _stride_w;
inc_w = stride_w * Delta::kW;
inc_h = stride_h * Delta::kH - stride_w * Delta::kW * (Iterations::kW - 1);
if (kAdvance == IteratorAdvance::kH) {
// Advance in the H dimension.
inc_d = 0;
} else if (kAdvance == IteratorAdvance::kW) {
// Advance in the W dimension.
inc_d = stride_w * Tile::kW - stride_h * Tile::kH;
} else {
// Advance in the D dimension.
inc_d = stride_d;
}
inc_advance = 0;
return 0;
}
CUTLASS_HOST_DEVICE int initialize() {
stride_d = 0;
stride_h = 0;
stride_w = 1;
inc_d = inc_h = inc_w = inc_advance = 0;
return 0;
}
};
CUTLASS_DEVICE bool valid(int d, int h, int w, int c) const { return true; }
//
// Static function members
//
template <typename PredicateIterator>
CUTLASS_DEVICE static void initialize_predicates(PredicateIterator predicate_it,
Coord<3> const &bounds,
Coord<3> const &offset = make_Coord(0, 0, 0)) {
for (int d = 0; d < Iterations::kD; ++d) {
bool enable_d = (d * Delta::kD + offset[0] < bounds[0]);
for (int h = 0; h < Iterations::kH; ++h) {
bool enable_h = (h * Delta::kH + offset[1] < bounds[1]);
for (int w = 0; w < Iterations::kW; ++w) {
bool enable_w = (w * Tile::kC * Delta::kW + offset[2] < bounds[2]);
predicate_it.set(d, h, w, 0, enable_d && enable_h && enable_w);
}
}
}
}
};
template <typename Traits_,
typename Scalar_,
IteratorAdvance::Kind Advance_ = IteratorAdvance::kH,
MemorySpace::Kind MemorySpace = MemorySpace::kGeneric,
typename Index_ = int,
typename FragmentElement_ = Scalar_,
IteratorFragment::Kind IteratorFragment_ = IteratorFragment::kScalar,
typename Skew_ = Shape<0, 0, 0, 0> >
struct TileLoadIterator : public TileIteratorBase<Traits_,
Scalar_,
Advance_,
MemorySpace,
Index_,
FragmentElement_,
IteratorFragment_,
Skew_> {
typedef TileIteratorBase<Traits_,
Scalar_,
Advance_,
MemorySpace,
Index_,
FragmentElement_,
IteratorFragment_,
Skew_>
Base;
typedef typename Base::Traits Traits;
typedef typename Base::Scalar Scalar;
typedef typename Base::FragmentElement FragmentElement;
static IteratorAdvance::Kind const kAdvance = Base::kAdvance;
static IteratorFragment::Kind const kIteratorFragment = Base::kIteratorFragment;
static MemorySpace::Kind const kMemorySpace = Base::kMemorySpace;
typedef typename Base::Index Index;
typedef typename Base::Skew Skew;
typedef typename Base::Tile Tile;
typedef typename Base::Delta Delta;
typedef typename Base::Iterations Iterations;
typedef typename Base::ThreadOffset ThreadOffset;
typedef typename Base::FragmentShape FragmentShape;
typedef typename Base::AccessType AccessType;
typedef typename Base::Fragment Fragment;
typedef typename Base::FragmentIterator FragmentIterator;
typedef typename Base::FragmentConstIterator FragmentConstIterator;
typedef typename Base::PredicateVector PredicateVector;
typedef typename Base::Storage SharedStorage;
typedef typename Base::Params BaseParams;
enum { kRequiresLoadFence = Tile::kD == 1 };
typedef Scalar const *Pointer;
struct Params : public BaseParams {
Scalar const *pointer;
CUTLASS_HOST_DEVICE
int initialize(SharedStorage const &storage) {
pointer = &storage[0];
return 0;
}
CUTLASS_HOST_DEVICE
int initialize(Scalar const *ptr, Index stride_d, Index stride_h, Index stride_w) {
Base::Params::initialize(stride_d, stride_h, stride_w);
pointer = ptr;
return 0;
}
CUTLASS_HOST_DEVICE
int initialize(Scalar const *ptr,
Index _stride_d,
Index _stride_h,
Index _stride_w,
Index _inc_d,
Index _inc_h,
Index _inc_w,
Index _inc_advance) {
pointer = ptr;
Base::Params::initialize(
_stride_d, _stride_h, _stride_w, _inc_d, _inc_h, _inc_w, _inc_advance);
return 0;
}
// Initializes params to default values
CUTLASS_HOST_DEVICE
int initialize() { return Base::Params::initialize(); }
};
//
// Data members
//
Params params;
Coord<4> thread_offset;
int stage;
//
// Static member functions
//
template <typename PredicateIterator>
CUTLASS_HOST_DEVICE void initialize_predicates(PredicateIterator predicate_it,
Coord<3> const &bounds,
Coord<3> const &block_offset = make_Coord(0,
0,
0)) {
Base::initialize_predicates(
predicate_it,
bounds,
block_offset + make_Coord(0, thread_offset[1], thread_offset[2] * Tile::kC));
}
//
// Methods
//
CUTLASS_HOST_DEVICE
TileLoadIterator() {}
CUTLASS_HOST_DEVICE
TileLoadIterator(Params const &_params,
Coord<3> const &block_offset = make_Coord(0, 0, 0),
ThreadOffset thread_offset_func = ThreadOffset())
: params(_params), stage(0) {
thread_offset = thread_offset_func();
Index block_offset_h = 0;
Index block_offset_w = 0;
if (kAdvance == IteratorAdvance::kH) {
block_offset_h = block_offset[1];
block_offset_w = block_offset[2];
} else {
block_offset_h = block_offset[2];
block_offset_w = block_offset[1];
}
params.pointer += block_offset[0] * params.stride_d +
(block_offset_h + thread_offset[1]) * params.stride_h +
(block_offset_w + thread_offset[2] * Tile::kC) / Tile::kC * params.stride_w;
}
CUTLASS_HOST_DEVICE
TileLoadIterator(Params const &,
SharedStorage &shared_storage,
Coord<3> const &block_offset = make_Coord(0, 0, 0),
ThreadOffset thread_offset_func = ThreadOffset())
: stage(0) {
int const offset = thread_offset_func()[2];
params.pointer = &shared_storage[offset];
}
CUTLASS_HOST_DEVICE
Scalar const *data() const { return params.pointer; }
CUTLASS_HOST_DEVICE void inc_d() { params.pointer += params.inc_d; }
CUTLASS_HOST_DEVICE void inc_h() { params.pointer += params.inc_h; }
CUTLASS_HOST_DEVICE void inc_w() { params.pointer += params.inc_w; }
CUTLASS_HOST_DEVICE void inc_advance() { params.pointer += params.inc_advance; }
CUTLASS_DEVICE void inc_stage() {
if (Tile::kD > 1) {
int const kStageSize = Tile::kH * Tile::kW * Tile::kC;
if (stage == Tile::kD - 1) {
params.pointer -= (Tile::kD - 1) * kStageSize;
stage = 0;
} else {
params.pointer += kStageSize;
stage = stage + 1;
}
}
}
public:
template <typename Fragment, typename PredicateIterator>
CUTLASS_HOST_DEVICE void load_post_increment(Fragment &fragment, PredicateIterator pred_it) {
FragmentIterator frag_iterator(fragment);
for (int d = 0; d < Iterations::kD; ++d) {
for (int h = 0; h < Iterations::kH; ++h) {
for (int w = 0; w < Iterations::kW; ++w, ++pred_it) {
if (*pred_it) {
Load<typename Fragment::Element, Tile::kC, kMemorySpace>::load(
reinterpret_cast<AccessType &>(frag_iterator.at(d, h, w, 0)), data(), 0);
}
if (w < Iterations::kW - 1) {
inc_w();
}
}
if (h < Iterations::kH - 1) {
inc_h();
}
}
if (d < Iterations::kD - 1) {
inc_d();
}
}
inc_advance();
}
template <typename Fragment>
CUTLASS_HOST_DEVICE void load_post_increment(Fragment &fragment) {
typename PredicateVector::TrivialIterator pred_it;
load_post_increment(fragment, pred_it);
}
template <typename Fragment, typename PredicateIterator>
CUTLASS_HOST_DEVICE void load(Fragment &fragment, PredicateIterator pred_it) const {
TileLoadIterator _load_it(*this);
_load_it.load_post_increment(fragment, pred_it);
}
template <typename Fragment>
CUTLASS_HOST_DEVICE void load(Fragment &fragment) const {
typename PredicateVector::TrivialIterator pred_it;
load(fragment, pred_it);
}
};
template <typename Traits_,
typename Scalar_,
IteratorAdvance::Kind Advance_ = IteratorAdvance::kH,
MemorySpace::Kind MemorySpace = MemorySpace::kGeneric,
typename Index_ = int,
typename FragmentElement_ = Scalar_,
IteratorFragment::Kind IteratorFragment_ = IteratorFragment::kScalar,
typename Skew_ = Shape<0, 0, 0, 0> >
struct TileStoreIterator : public TileIteratorBase<Traits_,
Scalar_,
Advance_,
MemorySpace,
Index_,
FragmentElement_,
IteratorFragment_,
Skew_> {
typedef TileIteratorBase<Traits_,
Scalar_,
Advance_,
MemorySpace,
Index_,
FragmentElement_,
IteratorFragment_,
Skew_>
Base;
typedef typename Base::Traits Traits;
typedef typename Base::Scalar Scalar;
typedef typename Base::FragmentElement FragmentElement;
static IteratorAdvance::Kind const kAdvance = Base::kAdvance;
static IteratorFragment::Kind const kIteratorFragment = Base::kIteratorFragment;
static MemorySpace::Kind const kMemorySpace = Base::kMemorySpace;
typedef typename Base::Index Index;
typedef typename Base::Skew Skew;
typedef typename Base::Tile Tile;
typedef typename Base::Delta Delta;
typedef typename Base::Iterations Iterations;
typedef typename Base::ThreadOffset ThreadOffset;
typedef typename Base::FragmentShape FragmentShape;
typedef typename Base::AccessType AccessType;
typedef typename Base::Fragment Fragment;
typedef typename Base::FragmentIterator FragmentIterator;
typedef typename Base::FragmentConstIterator FragmentConstIterator;
typedef typename Base::PredicateVector PredicateVector;
typedef typename Base::Storage SharedStorage;
typedef typename Base::Params BaseParams;
struct Params : public BaseParams {
Scalar *pointer;
CUTLASS_HOST_DEVICE
int initialize(SharedStorage &storage) {
pointer = &storage[0];
return 0;
}
CUTLASS_HOST_DEVICE
int initialize(Scalar *ptr, Index stride_d, Index stride_h, Index stride_w) {
Base::Params::initialize(stride_d, stride_h, stride_w);
pointer = ptr;
return 0;
}
CUTLASS_HOST_DEVICE
int initialize(Scalar *ptr,
Index _stride_d,
Index _stride_h,
Index _stride_w,
Index _inc_d,
Index _inc_h,
Index _inc_w,
Index _inc_advance) {
pointer = ptr;
Base::Params::initialize(
_stride_d, _stride_h, _stride_w, _inc_d, _inc_h, _inc_w, _inc_advance);
return 0;
}
CUTLASS_HOST_DEVICE
int initialize() { return Base::Params::initialize(); }
};
//
// Data members
//
Params params;
Coord<4> thread_offset;
int stage;
//
// Static member functions
//
template <typename PredicateIterator>
CUTLASS_HOST_DEVICE void initialize_predicates(PredicateIterator predicate_it,
Coord<3> const &bounds,
Coord<3> const &block_offset = make_Coord(0,
0,
0)) {
Base::initialize_predicates(
predicate_it,
bounds,
block_offset + make_Coord(0, thread_offset[1], thread_offset[2] * Tile::kC));
}
//
// Methods
//
CUTLASS_HOST_DEVICE
TileStoreIterator() {}
CUTLASS_HOST_DEVICE
TileStoreIterator(Params const &_params,
Coord<3> const &block_offset = make_Coord(0, 0, 0),
ThreadOffset thread_offset_func = ThreadOffset())
: params(_params), stage(0) {
thread_offset = thread_offset_func();
params.pointer += block_offset[0] * params.stride_d +
(block_offset[1] + thread_offset[1]) * params.stride_h +
(block_offset[2] + thread_offset[2] * Tile::kC) / Tile::kC * params.stride_w;
}
CUTLASS_HOST_DEVICE
TileStoreIterator(Params const &,
SharedStorage &shared_storage,
Coord<3> const &block_offset = make_Coord(0, 0, 0),
ThreadOffset thread_offset_func = ThreadOffset())
: stage(0) {
int const offset = thread_offset_func()[2];
params.pointer = &shared_storage[offset];
}
CUTLASS_HOST_DEVICE
Scalar *data() const { return params.pointer; }
CUTLASS_HOST_DEVICE void inc_d() { params.pointer += params.inc_d; }
CUTLASS_HOST_DEVICE void inc_h() { params.pointer += params.inc_h; }
CUTLASS_HOST_DEVICE void inc_w() { params.pointer += params.inc_w; }
CUTLASS_HOST_DEVICE void inc_advance() {}
CUTLASS_DEVICE void inc_stage() {
if (Tile::kD > 1) {
int const kStageSize = Tile::kH * Tile::kW * Tile::kC;
if (stage == Tile::kD - 1) {
params.pointer -= (Tile::kD - 1) * kStageSize;
stage = 0;
} else {
params.pointer += kStageSize;
stage = stage + 1;
}
}
}
public:
template <typename Fragment, typename PredicateIterator>
CUTLASS_HOST_DEVICE void store_post_increment(Fragment &fragment, PredicateIterator pred_it) {
FragmentIterator frag_iterator(fragment);
for (int d = 0; d < Iterations::kD; ++d) {
for (int h = 0; h < Iterations::kH; ++h) {
for (int w = 0; w < Iterations::kW; ++w, ++pred_it) {
if (*pred_it) {
Store<typename Fragment::Element, Tile::kC, kMemorySpace>::store(
reinterpret_cast<AccessType &>(frag_iterator.at(d, h, w, 0)), data(), 0);
}
if (w < Iterations::kW - 1) {
inc_w();
}
}
if (h < Iterations::kH - 1) {
inc_h();
}
}
if (d < Iterations::kD - 1) {
inc_d();
}
}
inc_advance();
}
template <typename Fragment>
CUTLASS_HOST_DEVICE void store_post_increment(Fragment &fragment) {
typename PredicateVector::TrivialIterator pred_it;
store_post_increment(fragment, pred_it);
}
template <typename Fragment, typename PredicateIterator>
CUTLASS_HOST_DEVICE void store(Fragment &fragment, PredicateIterator pred_it) const {
TileStoreIterator _store_it(*this);
_store_it.store_post_increment(fragment, pred_it);
}
template <typename Fragment>
CUTLASS_HOST_DEVICE void store(Fragment &fragment) const {
typename PredicateVector::TrivialIterator pred_it;
store(fragment, pred_it);
}
};
}
cutlass::TileIteratorBase::kFragmentSize
static int const kFragmentSize
The size of storage needed per fragment.
Definition: tile_iterator.h:149
cutlass::TileLoadIterator::kIteratorFragment
static IteratorFragment::Kind const kIteratorFragment
Specifies type of iterator fragment storage (Salar or WmmaMatrix)
Definition: tile_iterator.h:334
cutlass::TileStoreIterator::inc_advance
CUTLASS_HOST_DEVICE void inc_advance()
Increment in the next dimension.
Definition: tile_iterator.h:816
cutlass::TileIteratorBase< TileTraits_, TileTraits_::Scalar, IteratorAdvance::kH, MemorySpace::kGlobal, Index_ >::FragmentConstIterator
FragmentConstIterator< Fragment, Iterations, AccessType > FragmentConstIterator
The fragment const iterator.
Definition: tile_iterator.h:158
cutlass::TileStoreIterator::Base
TileIteratorBase< Traits_, Scalar_, Advance_, MemorySpace, Index_, FragmentElement_, IteratorFragment_, Skew_ > Base
Base class.
Definition: tile_iterator.h:637
cutlass::TileStoreIterator::Fragment
Base::Fragment Fragment
Fragment definition.
Definition: tile_iterator.h:682
cutlass::TileLoadIterator::FragmentIterator
Base::FragmentIterator FragmentIterator
Fragment iterator definition.
Definition: tile_iterator.h:367
cutlass
Definition: convert.h:33
cutlass::TileLoadIterator::Tile
Base::Tile Tile
Tile shape.
Definition: tile_iterator.h:346
cutlass::TileLoadIterator::load_post_increment
CUTLASS_HOST_DEVICE void load_post_increment(Fragment &fragment, PredicateIterator pred_it)
Loads a fragment and advances the iterator to the next tile.
Definition: tile_iterator.h:533
cutlass::TileLoadIterator::ThreadOffset
Base::ThreadOffset ThreadOffset
ThreadOffset functor.
Definition: tile_iterator.h:355
cutlass::TileStoreIterator::kAdvance
static IteratorAdvance::Kind const kAdvance
Specifies in which dimension post-increment accesses advance.
Definition: tile_iterator.h:649
cutlass::TileIteratorBase::FragmentShape
FragmentIterator::FragmentShape FragmentShape
The shape of the fragment.
Definition: tile_iterator.h:160
cutlass::TileIteratorBase::ThreadOffset
Traits::ThreadOffset ThreadOffset
Thread offset.
Definition: tile_iterator.h:140
cutlass::TileStoreIterator::kIteratorFragment
static IteratorFragment::Kind const kIteratorFragment
Specifies type of iterator fragment storage (Salar or WmmaMatrix)
Definition: tile_iterator.h:652
cutlass::TileIteratorBase::Skew
Skew_ Skew
Skew quantity.
Definition: tile_iterator.h:125
cutlass::TileStoreIterator::FragmentShape
Base::FragmentShape FragmentShape
Fragment type.
Definition: tile_iterator.h:676
cutlass::TileStoreIterator::Params::initialize
CUTLASS_HOST_DEVICE int initialize(SharedStorage &storage)
Initialize params to access storage object.
Definition: tile_iterator.h:706
cutlass::MemorySpace
Enum to specify which memory space data resides in.
Definition: load_store.h:39
cutlass::TileLoadIterator::Skew
Base::Skew Skew
Skew quantity.
Definition: tile_iterator.h:343
cutlass::IteratorAdvance::Kind
Kind
Definition: tile_iterator.h:62
cutlass::TileIteratorBase::Params::initialize
CUTLASS_HOST_DEVICE int initialize()
Definition: tile_iterator.h:227
cutlass::TileStoreIterator::Skew
Base::Skew Skew
Skew quantity.
Definition: tile_iterator.h:661
cutlass::make_Coord
CUTLASS_HOST_DEVICE Coord< 1 > make_Coord(int _0)
Helper to make a 2-element coordinate.
Definition: coord.h:241
cutlass::TileStoreIterator::SharedStorage
Base::Storage SharedStorage
Storage object which may be stored to.
Definition: tile_iterator.h:694
cutlass::TileTraits
A template defining Tile Traits Concept.
Definition: tile_iterator.h:77
cutlass::TileLoadIterator::data
CUTLASS_HOST_DEVICE Scalar const * data() const
Returns the current pointer.
Definition: tile_iterator.h:502
cutlass::TileLoadIterator::Base
TileIteratorBase< Traits_, Scalar_, Advance_, MemorySpace, Index_, FragmentElement_, IteratorFragment_, Skew_ > Base
Base class.
Definition: tile_iterator.h:319
cutlass::TileLoadIterator::TileLoadIterator
CUTLASS_HOST_DEVICE TileLoadIterator(Params const &, SharedStorage &shared_storage, Coord< 3 > const &block_offset=make_Coord(0, 0, 0), ThreadOffset thread_offset_func=ThreadOffset())
Constructs a tile load iterator.
Definition: tile_iterator.h:491
cutlass::TileLoadIterator::Params::initialize
CUTLASS_HOST_DEVICE int initialize(Scalar const *ptr, Index stride_d, Index stride_h, Index stride_w)
Initializes params to access a raw pointer.
Definition: tile_iterator.h:401
cutlass::TileLoadIterator::BaseParams
Base::Params BaseParams
IteratorBase parameters.
Definition: tile_iterator.h:379
cutlass::TileStoreIterator::params
Params params
Parameters structure.
Definition: tile_iterator.h:745
cutlass::Load::load
static CUTLASS_DEVICE void load(AccessType &dst, Scalar_ const *pointer, int offset)
The load function.
Definition: load_store.h:59
cutlass::TileLoadIterator::Params::initialize
CUTLASS_HOST_DEVICE int initialize(SharedStorage const &storage)
Initialize params to access storage object.
Definition: tile_iterator.h:394
cutlass::TileLoadIterator::kRequiresLoadFence
Definition: tile_iterator.h:382
cutlass::TileLoadIterator::Scalar
Base::Scalar Scalar
Scalar element.
Definition: tile_iterator.h:325
cutlass::TileLoadIterator::AccessType
Base::AccessType AccessType
Memory access type.
Definition: tile_iterator.h:361
cutlass::IteratorAdvance::kH
Definition: tile_iterator.h:62
cutlass::TileStoreIterator::store
CUTLASS_HOST_DEVICE void store(Fragment &fragment, PredicateIterator pred_it) const
Stores a fragment without advancing the iterator.
Definition: tile_iterator.h:869
cutlass::TileLoadIterator::kAdvance
static IteratorAdvance::Kind const kAdvance
Specifies in which dimension post-increment accesses advance.
Definition: tile_iterator.h:331
cutlass::TileLoadIterator::load_post_increment
CUTLASS_HOST_DEVICE void load_post_increment(Fragment &fragment)
Loads a fragment and advances the iterator to the next tile.
Definition: tile_iterator.h:561
cutlass::TileLoadIterator::TileLoadIterator
CUTLASS_HOST_DEVICE TileLoadIterator(Params const &_params, Coord< 3 > const &block_offset=make_Coord(0, 0, 0), ThreadOffset thread_offset_func=ThreadOffset())
Constructs a tile load iterator.
Definition: tile_iterator.h:468
cutlass::TileIteratorBase::valid
CUTLASS_DEVICE bool valid(int d, int h, int w, int c) const
Is the iterator valid?
Definition: tile_iterator.h:239
cutlass::TileTraits::Iterations
Iterations_ Iterations
Number of accesses performed.
Definition: tile_iterator.h:85
cutlass::TileStoreIterator::Params::initialize
CUTLASS_HOST_DEVICE int initialize(Scalar *ptr, Index stride_d, Index stride_h, Index stride_w)
Initializes params to access a raw pointer.
Definition: tile_iterator.h:713
cutlass::TileLoadIterator::params
Params params
Parameters structure.
Definition: tile_iterator.h:433
cutlass::PredicateVector::TrivialIterator
Iterator that always returns true.
Definition: predicate_vector.h:308
cutlass::TileStoreIterator::Scalar
Base::Scalar Scalar
Scalar element.
Definition: tile_iterator.h:643
cutlass::TileStoreIterator::Traits
Base::Traits Traits
concept TileTraits
Definition: tile_iterator.h:640
cutlass::MemorySpace::Kind
Kind
Definition: load_store.h:40
cutlass::TileIteratorBase::Params::stride_h
Index stride_h
Definition: tile_iterator.h:172
cutlass::TileStoreIterator::store_post_increment
CUTLASS_HOST_DEVICE void store_post_increment(Fragment &fragment)
Stores a fragment and advances to the next tile.
Definition: tile_iterator.h:862
cutlass::TileIteratorBase::Storage
Fragment< Scalar, ShapeCount< Tile >::kCount, kFragmentSize > Storage
The storage.
Definition: tile_iterator.h:152
cutlass::TileLoadIterator::Params::initialize
CUTLASS_HOST_DEVICE int initialize()
Definition: tile_iterator.h:425
cutlass::TileStoreIterator::inc_d
CUTLASS_HOST_DEVICE void inc_d()
Increment in the D dimension.
Definition: tile_iterator.h:807
cutlass::TileLoadIterator::Iterations
Base::Iterations Iterations
Iterations.
Definition: tile_iterator.h:352
cutlass::TileStoreIterator::Params::initialize
CUTLASS_HOST_DEVICE int initialize()
Initializes params to default values.
Definition: tile_iterator.h:737
cutlass::TileLoadIterator::FragmentConstIterator
Base::FragmentConstIterator FragmentConstIterator
Fragment const iterator definition.
Definition: tile_iterator.h:370
cutlass::TileIteratorBase::Index
Index_ Index
Index type.
Definition: tile_iterator.h:122
cutlass::Store::store
static CUTLASS_DEVICE void store(AccessType const &src, Scalar_ *pointer, int offset)
The store function.
Definition: load_store.h:136
cutlass::TileIteratorBase::Params::inc_h
Index inc_h
Definition: tile_iterator.h:176
predicate_vector.h
Defines container classes and iterators for managing a statically sized vector of boolean predicates...
cutlass::TileLoadIterator::SharedStorage
Base::Storage SharedStorage
Storage object that may be loaded from.
Definition: tile_iterator.h:376
cutlass::TileStoreIterator::Params
Parameters.
Definition: tile_iterator.h:700
cutlass::TileStoreIterator::initialize_predicates
CUTLASS_HOST_DEVICE void initialize_predicates(PredicateIterator predicate_it, Coord< 3 > const &bounds, Coord< 3 > const &block_offset=make_Coord(0, 0, 0))
Initializes a predicate vector.
Definition: tile_iterator.h:759
cutlass::TileLoadIterator
An iterator implementing Tile Load Iterator Concept for loading a tile from memory.
Definition: tile_iterator.h:302
cutlass::TileLoadIterator::Traits
Base::Traits Traits
concept TileTraits
Definition: tile_iterator.h:322
cutlass::TileIteratorBase::initialize_predicates
static CUTLASS_DEVICE void initialize_predicates(PredicateIterator predicate_it, Coord< 3 > const &bounds, Coord< 3 > const &offset=make_Coord(0, 0, 0))
Initializes a predicate vector.
Definition: tile_iterator.h:247
cutlass::TileStoreIterator::BaseParams
Base::Params BaseParams
IteratorBase parameters.
Definition: tile_iterator.h:697
cutlass::TileLoadIterator::FragmentElement
Base::FragmentElement FragmentElement
Fragment element.
Definition: tile_iterator.h:328
cutlass::TileIteratorBase::Tile
Traits::Tile Tile
Tile shape.
Definition: tile_iterator.h:128
cutlass::TileIteratorBase< TileTraits_, TileTraits_::Scalar, IteratorAdvance::kH, MemorySpace::kGlobal, Index_ >::FragmentIterator
FragmentIterator< Fragment, Iterations, AccessType > FragmentIterator
The fragment iterator.
Definition: tile_iterator.h:156
cutlass::TileStoreIterator::stage
int stage
The stage.
Definition: tile_iterator.h:751
cutlass::TileIteratorBase::Params::initialize
CUTLASS_HOST_DEVICE int initialize(Index _stride_d, Index _stride_h, Index _stride_w, Index _inc_d, Index _inc_h, Index _inc_w, Index _inc_advance)
Initializes params.
Definition: tile_iterator.h:183
cutlass::TileStoreIterator::AccessType
Base::AccessType AccessType
Memory access type.
Definition: tile_iterator.h:679
cutlass::TileStoreIterator::FragmentElement
Base::FragmentElement FragmentElement
Fragment element.
Definition: tile_iterator.h:646
cutlass::MemorySpace::kGeneric
Definition: load_store.h:41
cutlass::TileLoadIterator::inc_stage
CUTLASS_DEVICE void inc_stage()
Increment the stage.
Definition: tile_iterator.h:517
cutlass::IteratorFragment::Kind
Kind
Definition: tile_iterator.h:67
cutlass::TileLoadIterator::PredicateVector
Base::PredicateVector PredicateVector
Default predicate mask type.
Definition: tile_iterator.h:373
cutlass::TileLoadIterator::Params::initialize
CUTLASS_HOST_DEVICE int initialize(Scalar const *ptr, Index _stride_d, Index _stride_h, Index _stride_w, Index _inc_d, Index _inc_h, Index _inc_w, Index _inc_advance)
Initializes params.
Definition: tile_iterator.h:409
cutlass::TileStoreIterator::Params::initialize
CUTLASS_HOST_DEVICE int initialize(Scalar *ptr, Index _stride_d, Index _stride_h, Index _stride_w, Index _inc_d, Index _inc_h, Index _inc_w, Index _inc_advance)
Initializes params.
Definition: tile_iterator.h:721
cutlass::TileIteratorBase::FragmentElement
FragmentElement_ FragmentElement
Fragment element.
Definition: tile_iterator.h:110
cutlass::TileStoreIterator::Index
Base::Index Index
Index type.
Definition: tile_iterator.h:658
cutlass::TileStoreIterator::Params::pointer
Scalar * pointer
Pointer to memory.
Definition: tile_iterator.h:702
cutlass::TileIteratorBase::Params::inc_advance
Index inc_advance
Definition: tile_iterator.h:179
cutlass::IteratorFragment::kWmmaMatrix
Definition: tile_iterator.h:67
cutlass::FragmentIterator::FragmentShape
ShapeMul< Iterations, Shape< 1, 1, 1, kElementsPerAccess > >::Shape FragmentShape
The shape of the the fragment.
Definition: fragment.h:185
cutlass::TileIteratorBase::Params::stride_w
Index stride_w
Definition: tile_iterator.h:173
cutlass::TileLoadIterator::TileLoadIterator
CUTLASS_HOST_DEVICE TileLoadIterator()
Default constructor.
Definition: tile_iterator.h:464
load_store.h
Defines abstractions for efficiently loading and storing vectors to memory.
cutlass::TileLoadIterator::Params::pointer
Scalar const * pointer
Pointer to memory.
Definition: tile_iterator.h:390
cutlass::TileStoreIterator::TileStoreIterator
CUTLASS_HOST_DEVICE TileStoreIterator(Params const &_params, Coord< 3 > const &block_offset=make_Coord(0, 0, 0), ThreadOffset thread_offset_func=ThreadOffset())
Constructs a tile store iterator.
Definition: tile_iterator.h:780
CUTLASS_HOST_DEVICE
#define CUTLASS_HOST_DEVICE
Definition: cutlass.h:46
cutlass::TileStoreIterator::thread_offset
Coord< 4 > thread_offset
Offset of an individual lane from the start of the tile.
Definition: tile_iterator.h:748
cutlass::TileIteratorBase::Iterations
Traits::Iterations Iterations
Iterations.
Definition: tile_iterator.h:137
cutlass::TileIteratorBase::kAccessSize
static int const kAccessSize
The number of scalars accessed per load/store.
Definition: tile_iterator.h:143
cutlass::TileTraits::Tile
Tile_ Tile
Shape of the tile.
Definition: tile_iterator.h:79
cutlass::TileTraits::Delta
Delta_ Delta
Number of steps between accesses along each dimension.
Definition: tile_iterator.h:82
cutlass::TileIteratorBase::Params::initialize
CUTLASS_HOST_DEVICE int initialize(Index _stride_d, Index _stride_h, Index _stride_w)
Definition: tile_iterator.h:203
cutlass::TileIteratorBase::Params::stride_d
Index stride_d
Definition: tile_iterator.h:171
cutlass::TileLoadIterator::inc_advance
CUTLASS_HOST_DEVICE void inc_advance()
Increment in the next dimension.
Definition: tile_iterator.h:514
cutlass::Vector
Definition: vector.h:61
cutlass::TileLoadIterator::Delta
Base::Delta Delta
Delta.
Definition: tile_iterator.h:349
cutlass::TileStoreIterator::Tile
Base::Tile Tile
Tile shape.
Definition: tile_iterator.h:664
cutlass::Shape
A Shape implementing Layout Concept describing the dimensions of a cube.
Definition: shape.h:64
cutlass::TileLoadIterator::FragmentShape
Base::FragmentShape FragmentShape
Fragment type.
Definition: tile_iterator.h:358
cutlass::IteratorAdvance
Specifies dimension in which post-increment accesses advance.
Definition: tile_iterator.h:61
cutlass::TileStoreIterator::kMemorySpace
static MemorySpace::Kind const kMemorySpace
Source or destination memory space.
Definition: tile_iterator.h:655
cutlass::TileLoadIterator::inc_w
CUTLASS_HOST_DEVICE void inc_w()
Increment in the W dimension.
Definition: tile_iterator.h:511
cutlass::Coord
Statically-sized array specifying Coords within a tensor.
Definition: coord.h:48
cutlass::TileIteratorBase::ImmediateOffsetStrides
Traits::ImmediateOffsetStrides ImmediateOffsetStrides
The strides in each dimension between different loads/stores.
Definition: tile_iterator.h:134
cutlass::TileLoadIterator::Fragment
Base::Fragment Fragment
Fragment definition.
Definition: tile_iterator.h:364
cutlass::TileStoreIterator::Iterations
Base::Iterations Iterations
Iterations.
Definition: tile_iterator.h:670
vector.h
Defines a 1D vector of elements held in the registers of each thread.
cutlass::TileIteratorBase
Iterator for accessing a stripmined tile in memory.
Definition: tile_iterator.h:102
cutlass::TileIteratorBase::kIteratorFragment
static IteratorFragment::Kind const kIteratorFragment
Specifies iterator storage fragment type (Scalar or WmmaMatrix)
Definition: tile_iterator.h:116
cutlass::TileStoreIterator::Delta
Base::Delta Delta
Delta.
Definition: tile_iterator.h:667
cutlass::IteratorAdvance::kD
Definition: tile_iterator.h:62
cutlass::TileStoreIterator::data
CUTLASS_HOST_DEVICE Scalar * data() const
Returns the current pointer.
Definition: tile_iterator.h:804
cutlass::TileTraits::ThreadOffset
ThreadOffset_ ThreadOffset
Functor that returns the logical coordinate of each entity&#39;s initial offset in the tile...
Definition: tile_iterator.h:88
cutlass::TileIteratorBase::AccessType
Vectorize< FragmentElement, kAccessSize >::Type AccessType
The elements loaded/store by one instruction.
Definition: tile_iterator.h:146
cutlass::TileLoadIterator::inc_d
CUTLASS_HOST_DEVICE void inc_d()
Increment in the D dimension.
Definition: tile_iterator.h:505
cutlass::TileStoreIterator::inc_h
CUTLASS_HOST_DEVICE void inc_h()
Increment in the H dimension.
Definition: tile_iterator.h:810
cutlass::TileStoreIterator::store
CUTLASS_HOST_DEVICE void store(Fragment &fragment) const
Stores a fragment without advancing the iterator.
Definition: tile_iterator.h:876
cutlass::TileLoadIterator::inc_h
CUTLASS_HOST_DEVICE void inc_h()
Increment in the H dimension.
Definition: tile_iterator.h:508
cutlass::TileLoadIterator::Params
Parameters.
Definition: tile_iterator.h:388
cutlass::TileLoadIterator::kMemorySpace
static MemorySpace::Kind const kMemorySpace
Source or destination memory space.
Definition: tile_iterator.h:337
cutlass::TileStoreIterator::ThreadOffset
Base::ThreadOffset ThreadOffset
ThreadOffset functor.
Definition: tile_iterator.h:673
cutlass::TileIteratorBase::kMemorySpace
static MemorySpace::Kind const kMemorySpace
Source or destination memory space.
Definition: tile_iterator.h:119
cutlass::TileStoreIterator::FragmentIterator
Base::FragmentIterator FragmentIterator
Fragment iterator definition.
Definition: tile_iterator.h:685
cutlass::TileLoadIterator::load
CUTLASS_HOST_DEVICE void load(Fragment &fragment, PredicateIterator pred_it) const
Loads a fragment without advancing the iterator..
Definition: tile_iterator.h:568
cutlass::TileStoreIterator::TileStoreIterator
CUTLASS_HOST_DEVICE TileStoreIterator(Params const &, SharedStorage &shared_storage, Coord< 3 > const &block_offset=make_Coord(0, 0, 0), ThreadOffset thread_offset_func=ThreadOffset())
Constructs a tile store iterator.
Definition: tile_iterator.h:793
cutlass::TileLoadIterator::initialize_predicates
CUTLASS_HOST_DEVICE void initialize_predicates(PredicateIterator predicate_it, Coord< 3 > const &bounds, Coord< 3 > const &block_offset=make_Coord(0, 0, 0))
Initializes a predicate vector.
Definition: tile_iterator.h:447
cutlass::TileIteratorBase< TileTraits_, TileTraits_::Scalar, IteratorAdvance::kH, MemorySpace::kGlobal, Index_ >::Fragment
Fragment< FragmentElement, ShapeCount< Iterations >::kCount *kAccessSize > Fragment
The fragment.
Definition: tile_iterator.h:154
cutlass::TileLoadIterator::load
CUTLASS_HOST_DEVICE void load(Fragment &fragment) const
Loads a fragment without advancing the iterator..
Definition: tile_iterator.h:575
cutlass::IteratorAdvance::kW
Definition: tile_iterator.h:62
cutlass::TileIteratorBase::kAdvance
static IteratorAdvance::Kind const kAdvance
Specifies dimension in which post-increment accesses advance.
Definition: tile_iterator.h:113
cutlass::TileIteratorBase::Params::inc_w
Index inc_w
Definition: tile_iterator.h:177
cutlass::TileLoadIterator::thread_offset
Coord< 4 > thread_offset
Offset of an individual lane from the start of the tile.
Definition: tile_iterator.h:436
cutlass::TileIteratorBase::Delta
Traits::Delta Delta
Distance along each dimension.
Definition: tile_iterator.h:131
cutlass::TileLoadIterator::stage
int stage
Stage argument enables wrapping after some number of tiles have been loaded.
Definition: tile_iterator.h:439
cutlass::TileStoreIterator::FragmentConstIterator
Base::FragmentConstIterator FragmentConstIterator
Fragment const iterator definition.
Definition: tile_iterator.h:688
cutlass::TileStoreIterator::TileStoreIterator
CUTLASS_HOST_DEVICE TileStoreIterator()
Default constructor.
Definition: tile_iterator.h:776
cutlass::TileStoreIterator::PredicateVector
Base::PredicateVector PredicateVector
Default predicate mask type.
Definition: tile_iterator.h:691
cutlass::TileLoadIterator::Pointer
Scalar const * Pointer
The pointer type.
Definition: tile_iterator.h:385
fragment.h
Defines Fragment, a statically-sized array for storing parts of matrices within a thread&#39;s registers...
cutlass::TileIteratorBase::Params
Parameters to the iterator.
Definition: tile_iterator.h:170
cutlass::TileLoadIterator::Index
Base::Index Index
Index type.
Definition: tile_iterator.h:340
cutlass::TileStoreIterator::inc_stage
CUTLASS_DEVICE void inc_stage()
Increment the stage.
Definition: tile_iterator.h:819
cutlass::TileStoreIterator::store_post_increment
CUTLASS_HOST_DEVICE void store_post_increment(Fragment &fragment, PredicateIterator pred_it)
Stores a fragment and advances to the next tile.
Definition: tile_iterator.h:835
cutlass::TileStoreIterator::inc_w
CUTLASS_HOST_DEVICE void inc_w()
Increment in the W dimension.
Definition: tile_iterator.h:813
cutlass::TileIteratorBase< TileTraits_, TileTraits_::Scalar, IteratorAdvance::kH, MemorySpace::kGlobal, Index_ >::PredicateVector
PredicateVector< ShapeCount< Iterations >::kCount > PredicateVector
Default predicate mask type.
Definition: tile_iterator.h:163
cutlass::IteratorFragment::kScalar
Definition: tile_iterator.h:67
cutlass::TileIteratorBase::Scalar
Scalar_ Scalar
Scalar element.
Definition: tile_iterator.h:107
cutlass::IteratorFragment
Specifies whether iterator storage fragment consists of Scalar values or WMMA matrix.
Definition: tile_iterator.h:66
cutlass::TileIteratorBase::Params::inc_d
Index inc_d
Definition: tile_iterator.h:175
cutlass::TileStoreIterator
An iterator implementing Tile Store Iterator Concept for storing a tile to memory.
Definition: tile_iterator.h:620
cutlass::TileIteratorBase::Traits
Traits_ Traits
concept TileTraits
Definition: tile_iterator.h:104