gemm_global_stream.h
Go to the documentation of this file.
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#pragma once
#include <cutlass/convert.h>
#include <cutlass/gemm/gemm_global_tile.h>
#include <cutlass/iterator_access.h>
namespace cutlass {
namespace gemm {
template <
typename LoadIterator_,
typename StoreIterator_,
typename Transformer_>
struct GlobalLoadStreamBase {
typedef LoadIterator_ LoadIterator;
typedef Transformer_ Transformer;
typedef StoreIterator_ StoreIterator;
typedef typename LoadIterator::Fragment FetchedFragment;
typedef typename Transformer::OutputFragment TransformedFragment;
static_assert((platform::is_same<FetchedFragment, typename Transformer::InputFragment>::value),
"");
typedef TransformedFragment Fragment;
static_assert((platform::is_same<TransformedFragment, typename StoreIterator::Fragment>::value),
"");
static MatrixLayout::Kind const kLayout = LoadIterator::kLayout;
typedef typename LoadIterator::Scalar Scalar;
typedef typename LoadIterator::Pointer Pointer;
typedef typename LoadIterator::Index Index;
struct Params {
// The load iterator.
typename LoadIterator::Params load_iterator;
// The store iterator.
typename StoreIterator::Params store_iterator;
CUTLASS_HOST_DEVICE int initialize(Pointer pointer, Index ld) {
int error_code = load_iterator.initialize(pointer, ld);
if (error_code) {
return error_code;
}
return store_iterator.initialize();
}
};
typedef typename StoreIterator::SharedStorage SharedStoreStorage;
union SharedStorage {
// The load iterator.
typename LoadIterator::SharedStorage load_iterator;
// The store iterator.
SharedStoreStorage store_iterator;
};
CUTLASS_DEVICE GlobalLoadStreamBase(Params const& params,
SharedStorage& shared_storage,
Coord<3> const bounds,
Coord<3> const& block)
: load_iterator(params.load_iterator, bounds, block),
transformer(),
store_iterator(params.store_iterator, shared_storage.store_iterator)
{
fetched_fragment.clear();
}
CUTLASS_DEVICE void copy() { iterator_load(load_iterator, fetched_fragment); }
CUTLASS_DEVICE void commit() {
transformer.transform(fetched_fragment, transformed_fragment);
iterator_store(store_iterator, transformed_fragment);
store_iterator.inc_stage();
}
CUTLASS_DEVICE void residue(Index k, bool skip_clear = false) {
load_iterator.residue(k);
if (!skip_clear) {
fetched_fragment.clear();
}
}
LoadIterator load_iterator;
FetchedFragment fetched_fragment;
Transformer transformer;
TransformedFragment transformed_fragment;
StoreIterator store_iterator;
};
template <
typename LoadIterator_,
typename StoreIterator_,
typename Transformer_ = Copy<typename LoadIterator_::Fragment> >
struct GlobalLoadStream : public GlobalLoadStreamBase<LoadIterator_, StoreIterator_, Transformer_> {
typedef GlobalLoadStreamBase<LoadIterator_, StoreIterator_, Transformer_> Base;
CUTLASS_DEVICE GlobalLoadStream(typename Base::Params const& params,
typename Base::SharedStorage& shared_storage,
Coord<3> const& bounds,
Coord<3> const& block)
: Base(params, shared_storage, bounds, block) {}
};
} // namespace gemm
} // namespace cutlass
cutlass::gemm::GlobalLoadStreamBase::kLayout
static MatrixLayout::Kind const kLayout
Make sure the transformed fragment is the same as the store fragment.
Definition: gemm_global_stream.h:71
cutlass::gemm::GlobalLoadStreamBase::Params::store_iterator
StoreIterator::Params store_iterator
Definition: gemm_global_stream.h:84
cutlass
Definition: convert.h:33
gemm_global_tile.h
Defines iterators for efficiently loading and storing to global memory.
cutlass::gemm::GlobalLoadStreamBase::Transformer
Transformer_ Transformer
The transformer.
Definition: gemm_global_stream.h:53
cutlass::gemm::GlobalLoadStreamBase::StoreIterator
StoreIterator_ StoreIterator
The store iterator to write to shared memory.
Definition: gemm_global_stream.h:55
cutlass::platform::is_same
std::is_same (false specialization)
Definition: platform.h:412
cutlass::gemm::GlobalLoadStreamBase::SharedStoreStorage
StoreIterator::SharedStorage SharedStoreStorage
The amount of storage in shared memory needed to store the tile.
Definition: gemm_global_stream.h:98
cutlass::gemm::GlobalLoadStreamBase::Fragment
TransformedFragment Fragment
Make sure the fragments match.
Definition: gemm_global_stream.h:63
cutlass::gemm::GlobalLoadStreamBase::transformed_fragment
TransformedFragment transformed_fragment
The fragment to convert the data after it has been fetched from shared memory.
Definition: gemm_global_stream.h:146
cutlass::gemm::GlobalLoadStreamBase::residue
CUTLASS_DEVICE void residue(Index k, bool skip_clear=false)
Execute the residue code.
Definition: gemm_global_stream.h:132
cutlass::Copy
Definition: convert.h:69
cutlass::gemm::GlobalLoadStreamBase::Params::initialize
CUTLASS_HOST_DEVICE int initialize(Pointer pointer, Index ld)
Setup the params.
Definition: gemm_global_stream.h:87
cutlass::gemm::GlobalLoadStreamBase::load_iterator
LoadIterator load_iterator
The iterator.
Definition: gemm_global_stream.h:140
cutlass::gemm::GlobalLoadStreamBase::Params::load_iterator
LoadIterator::Params load_iterator
Definition: gemm_global_stream.h:82
cutlass::gemm::GlobalLoadStream
Definition: gemm_global_stream.h:161
iterator_access.h
Free functions for loading and storing to implementations of tile iteartor concepts.
cutlass::gemm::GlobalLoadStreamBase::SharedStorage::load_iterator
LoadIterator::SharedStorage load_iterator
Definition: gemm_global_stream.h:103
cutlass::gemm::GlobalLoadStream::GlobalLoadStream
CUTLASS_DEVICE GlobalLoadStream(typename Base::Params const &params, typename Base::SharedStorage &shared_storage, Coord< 3 > const &bounds, Coord< 3 > const &block)
Ctor.
Definition: gemm_global_stream.h:166
cutlass::gemm::GlobalLoadStreamBase
Definition: gemm_global_stream.h:49
cutlass::gemm::GlobalLoadStreamBase::store_iterator
StoreIterator store_iterator
The store iterator.
Definition: gemm_global_stream.h:148
cutlass::gemm::GlobalLoadStreamBase::Pointer
LoadIterator::Pointer Pointer
The pointer.
Definition: gemm_global_stream.h:75
cutlass::gemm::GlobalLoadStreamBase::SharedStorage::store_iterator
SharedStoreStorage store_iterator
Definition: gemm_global_stream.h:105
cutlass::gemm::GlobalLoadStreamBase::TransformedFragment
Transformer::OutputFragment TransformedFragment
The fragment that is obtained after the transformation by the transformer.
Definition: gemm_global_stream.h:60
cutlass::gemm::GlobalLoadStreamBase::Scalar
LoadIterator::Scalar Scalar
The scalar type of the iterator.
Definition: gemm_global_stream.h:73
CUTLASS_HOST_DEVICE
#define CUTLASS_HOST_DEVICE
Definition: cutlass.h:46
static_assert
#define static_assert(__e, __m)
Definition: platform.h:145
cutlass::gemm::GlobalLoadStreamBase::Index
LoadIterator::Index Index
The index.
Definition: gemm_global_stream.h:77
cutlass::gemm::GlobalLoadStreamBase::transformer
Transformer transformer
The transformer.
Definition: gemm_global_stream.h:144
cutlass::gemm::GlobalLoadStream::Base
GlobalLoadStreamBase< LoadIterator_, StoreIterator_, Transformer_ > Base
The base class.
Definition: gemm_global_stream.h:163
cutlass::Coord
Statically-sized array specifying Coords within a tensor.
Definition: coord.h:48
cutlass::gemm::GlobalLoadStreamBase::FetchedFragment
LoadIterator::Fragment FetchedFragment
The fragment that is copied from shared memory.
Definition: gemm_global_stream.h:58
cutlass::gemm::GlobalLoadStreamBase::SharedStorage
The storage in shared memory needed by that stream.
Definition: gemm_global_stream.h:101
cutlass::iterator_store
CUTLASS_HOST_DEVICE void iterator_store(OutputIterator &iterator, Fragment &fragment)
Stores a fragment to an output iterator.
Definition: iterator_access.h:193
cutlass::gemm::GlobalLoadStreamBase::fetched_fragment
FetchedFragment fetched_fragment
The fragment to fetch from shared memory.
Definition: gemm_global_stream.h:142
cutlass::MatrixLayout::Kind
Kind
Definition: matrix_traits.h:36
cutlass::gemm::GlobalLoadStreamBase::LoadIterator
LoadIterator_ LoadIterator
The load iterator.
Definition: gemm_global_stream.h:51
cutlass::iterator_load
CUTLASS_HOST_DEVICE void iterator_load(InputIterator &iterator, Fragment &fragment)
Loads a fragment from an input iterator.
Definition: iterator_access.h:41
cutlass::gemm::GlobalLoadStreamBase::commit
CUTLASS_DEVICE void commit()
Commit the data.
Definition: gemm_global_stream.h:125
cutlass::gemm::GlobalLoadStreamBase::copy
CUTLASS_DEVICE void copy()
Load the data from shared memory to the fetch fragment.
Definition: gemm_global_stream.h:122
cutlass::gemm::GlobalLoadStreamBase::GlobalLoadStreamBase
CUTLASS_DEVICE GlobalLoadStreamBase(Params const &params, SharedStorage &shared_storage, Coord< 3 > const bounds, Coord< 3 > const &block)
Ctor.
Definition: gemm_global_stream.h:109
convert.h
Defines conversion operations among Fragments of different base type.
cutlass::gemm::GlobalLoadStreamBase::Params
The params.
Definition: gemm_global_stream.h:80