fragment.h
Go to the documentation of this file.
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#pragma once
#include <assert.h>
#include <cutlass/shape.h>
#include <cutlass/util/cutlass_math.h>
#include <cutlass/vector.h>
namespace cutlass {
template <int kAlignment_>
struct StorageType {
typedef uint64_t Type;
};
template <>
struct StorageType<4> {
typedef uint32_t Type;
};
template <>
struct StorageType<2> {
typedef uint16_t Type;
};
template <>
struct StorageType<1> {
typedef uint8_t Type;
};
template <typename Element_, int kElements_, size_t kAlignment_ = 16>
struct Fragment : public AlignedStruct<kAlignment_> {
static_assert(kAlignment_ == 16 || kAlignment_ >= sizeof(Element_), "Alignment is too small");
static_assert(is_pow2<kAlignment_>::value, "Alignment must be a power of two");
typedef Fragment<Element_, kElements_> This_;
typedef Element_ Element;
static int const kElements = kElements_;
CUTLASS_DEVICE void clear() {
// Avoid element-wise access for sub 32b element type
if (kAlignment_ >= 8 && (kElements * sizeof(Element)) % 8 == 0) {
uint64_t* ptr = reinterpret_cast<uint64_t*>(storage);
for (int i = 0; i < (kElements * sizeof(Element)) / 8; ++i) {
ptr[i] = uint64_t(0);
}
} else if (kAlignment_ >= 4 && (kElements * sizeof(Element)) % 4 == 0) {
uint32_t* ptr = reinterpret_cast<uint32_t*>(storage);
for (int i = 0; i < (kElements * sizeof(Element)) / 4; ++i) {
ptr[i] = uint32_t(0);
}
} else if (kAlignment_ >= 2 && (kElements * sizeof(Element)) % 2 == 0) {
uint16_t* ptr = reinterpret_cast<uint16_t*>(storage);
for (int i = 0; i < (kElements * sizeof(Element)) / 2; ++i) {
ptr[i] = uint16_t(0);
}
} else {
for (int i = 0; i < kElements; ++i) {
storage[i] = 0;
}
}
}
CUTLASS_DEVICE Element& operator[](int i) {
assert(i < kElements_);
return reinterpret_cast< Element*>(storage)[i];
}
CUTLASS_DEVICE Element const& operator[](int i) const {
assert(i < kElements_);
return reinterpret_cast< Element const*>(storage)[i];
}
private:
typedef typename StorageType<kAlignment_>::Type StorageType;
static int const kStorageCount =
(sizeof(Element_) * kElements_ + sizeof(StorageType) - 1) / sizeof(StorageType);
StorageType storage[kStorageCount];
static_assert(sizeof(StorageType) <= kAlignment_, "StorageType is too big for given alignment");
};
template <typename Fragment_, typename Iterations_, typename AccessType_>
struct FragmentIterator {
typedef FragmentIterator<Fragment_, Iterations_, AccessType_> This_;
typedef Fragment_ Fragment;
typedef Iterations_ Iterations;
typedef AccessType_ AccessType;
typedef typename Fragment::Element Element;
static int const kElementsPerAccess = (int)(sizeof(AccessType) / sizeof(Element));
typedef typename ShapeMul<Iterations, Shape<1, 1, 1, kElementsPerAccess> >::Shape FragmentShape;
typedef typename ShapeStrides<FragmentShape>::Shape Strides;
template <typename OtherFragment_>
CUTLASS_DEVICE FragmentIterator(OtherFragment_& fragment, int offset = 0)
: pointer(reinterpret_cast<Element*>(&fragment[offset])) {
static_assert(OtherFragment_::kElements >= Fragment::kElements, "");
}
CUTLASS_DEVICE AccessType const& at(int d, int h, int w, int c = 0) const {
int const imm = ComputeOffsetFromStrides<Strides>::get(d, h, w, c);
return reinterpret_cast< AccessType const&>(pointer[imm]);
}
CUTLASS_DEVICE AccessType& at(int d, int h, int w, int c = 0) {
int const imm = ComputeOffsetFromStrides<Strides>::get(d, h, w, c);
return reinterpret_cast< AccessType&>(pointer[imm]);
}
CUTLASS_DEVICE AccessType const& operator[](int i) const {
return reinterpret_cast< AccessType const&>(pointer[i * kElementsPerAccess]);
}
CUTLASS_DEVICE AccessType& operator[](int i) {
return reinterpret_cast< AccessType&>(pointer[i * kElementsPerAccess]);
}
CUTLASS_DEVICE bool valid(int d, int h, int w, int c) const { return true; }
Element* pointer;
};
template <typename Fragment_, typename Iterations_, typename AccessType_>
struct FragmentConstIterator {
typedef FragmentIterator<Fragment_, Iterations_, AccessType_> This_;
typedef Fragment_ Fragment;
typedef Iterations_ Iterations;
typedef AccessType_ AccessType;
typedef typename Fragment::Element Element;
static int const kElementsPerAccess = (int)(sizeof(AccessType) / sizeof(Element));
typedef typename ShapeMul<Iterations, Shape<1, 1, 1, kElementsPerAccess> >::Shape FragmentShape;
typedef typename ShapeStrides<FragmentShape>::Shape IterationsStrides;
template <typename OtherFragment_>
CUTLASS_DEVICE FragmentConstIterator(OtherFragment_& fragment, int offset = 0)
: pointer(reinterpret_cast<Element const*>(&fragment[offset])) {
static_assert(OtherFragment_::kElements >= Fragment::kElements, "");
}
CUTLASS_DEVICE FragmentConstIterator(
FragmentIterator<Fragment_, Iterations_, AccessType_> const& rhs_)
: pointer(reinterpret_cast<Element const*>(rhs_.offset)) {}
CUTLASS_DEVICE AccessType const& at(int d, int h, int w, int c = 0) const {
int const imm = ComputeOffsetFromStrides<IterationsStrides>::get(d, h, w, c);
return reinterpret_cast< AccessType const&>(pointer[imm]);
}
CUTLASS_DEVICE AccessType const& operator[](int i) const {
return reinterpret_cast< AccessType const&>(pointer[i * kElementsPerAccess]);
}
CUTLASS_DEVICE bool valid(int d, int h, int w, int c) const { return true; }
Element const* pointer;
};
} // namespace cutlass
cutlass::Fragment::clear
CUTLASS_DEVICE void clear()
Clear a fragment.
Definition: fragment.h:113
cutlass
Definition: convert.h:33
cutlass::Fragment::operator[]
CUTLASS_DEVICE Element & operator[](int i)
The accessor.
Definition: fragment.h:138
cutlass::FragmentIterator::at
CUTLASS_DEVICE AccessType & at(int d, int h, int w, int c=0)
The accessor.
Definition: fragment.h:203
cutlass::AlignedStruct
Definition: vector.h:41
cutlass::FragmentConstIterator
Definition: fragment.h:228
cutlass::FragmentConstIterator::operator[]
CUTLASS_DEVICE AccessType const & operator[](int i) const
The accessor.
Definition: fragment.h:265
cutlass::ShapeStrides::Shape
Shape< Shape_::kH *Shape_::kW *Shape_::kC, Shape_::kW *Shape_::kC, Shape_::kC, 1 > Shape
Definition: shape.h:155
cutlass::Fragment
A template defining Fragment Concept.
Definition: fragment.h:99
cutlass::FragmentIterator::Element
Fragment::Element Element
The element.
Definition: fragment.h:181
cutlass::FragmentConstIterator::kElementsPerAccess
static int const kElementsPerAccess
The number of elements per access.
Definition: fragment.h:241
cutlass::FragmentIterator::Fragment
Fragment_ Fragment
The fragment.
Definition: fragment.h:174
cutlass::FragmentConstIterator::Fragment
Fragment_ Fragment
The fragment.
Definition: fragment.h:232
cutlass::FragmentIterator::operator[]
CUTLASS_DEVICE AccessType & operator[](int i)
The accessor.
Definition: fragment.h:214
cutlass::FragmentConstIterator::Element
Fragment::Element Element
The element.
Definition: fragment.h:239
cutlass::FragmentConstIterator::IterationsStrides
ShapeStrides< FragmentShape >::Shape IterationsStrides
The linear strides for iterations.
Definition: fragment.h:245
cutlass::FragmentConstIterator::valid
CUTLASS_DEVICE bool valid(int d, int h, int w, int c) const
Is the iterator valid?
Definition: fragment.h:270
cutlass::FragmentIterator::FragmentIterator
CUTLASS_DEVICE FragmentIterator(OtherFragment_ &fragment, int offset=0)
Ctor.
Definition: fragment.h:191
cutlass::Fragment::This_
Fragment< Element_, kElements_ > This_
Make sure the alignment makes sense wrt the size of elements.
Definition: fragment.h:101
cutlass::FragmentIterator::This_
FragmentIterator< Fragment_, Iterations_, AccessType_ > This_
This class.
Definition: fragment.h:172
cutlass::FragmentConstIterator::FragmentShape
ShapeMul< Iterations, Shape< 1, 1, 1, kElementsPerAccess > >::Shape FragmentShape
The shape of the the fragment.
Definition: fragment.h:243
cutlass_math.h
Math utilities.
cutlass::StorageType
Definition: fragment.h:76
cutlass::StorageType< 4 >::Type
uint32_t Type
Definition: fragment.h:81
cutlass::StorageType< 1 >::Type
uint8_t Type
Definition: fragment.h:89
cutlass::ComputeOffsetFromStrides::get
static CUTLASS_DEVICE int get(int d, int h, int w, int c)
Definition: shape.h:211
cutlass::FragmentIterator::pointer
Element * pointer
The pointer.
Definition: fragment.h:222
cutlass::FragmentConstIterator::AccessType
AccessType_ AccessType
The access type.
Definition: fragment.h:236
cutlass::ShapeMul
Definition: shape.h:118
cutlass::FragmentIterator::FragmentShape
ShapeMul< Iterations, Shape< 1, 1, 1, kElementsPerAccess > >::Shape FragmentShape
The shape of the the fragment.
Definition: fragment.h:185
cutlass::FragmentIterator
A template defining Fragment Iterator Concept.
Definition: fragment.h:170
cutlass::Fragment::kElements
static int const kElements
The number of elements.
Definition: fragment.h:110
cutlass::Fragment::operator[]
CUTLASS_DEVICE Element const & operator[](int i) const
The accessor.
Definition: fragment.h:144
cutlass::FragmentConstIterator::Iterations
Iterations_ Iterations
The number of iterations.
Definition: fragment.h:234
static_assert
#define static_assert(__e, __m)
Definition: platform.h:145
cutlass::FragmentIterator::Iterations
Iterations_ Iterations
The number of iterations.
Definition: fragment.h:176
cutlass::Shape
A Shape implementing Layout Concept describing the dimensions of a cube.
Definition: shape.h:64
cutlass::FragmentConstIterator::at
CUTLASS_DEVICE AccessType const & at(int d, int h, int w, int c=0) const
The accessor.
Definition: fragment.h:259
cutlass::Fragment::Element
Element_ Element
The element.
Definition: fragment.h:108
cutlass::FragmentConstIterator::This_
FragmentIterator< Fragment_, Iterations_, AccessType_ > This_
This class.
Definition: fragment.h:230
cutlass::FragmentIterator::operator[]
CUTLASS_DEVICE AccessType const & operator[](int i) const
The accessor.
Definition: fragment.h:209
cutlass::StorageType< 2 >::Type
uint16_t Type
Definition: fragment.h:85
vector.h
Defines a 1D vector of elements held in the registers of each thread.
cutlass::FragmentConstIterator::FragmentConstIterator
CUTLASS_DEVICE FragmentConstIterator(FragmentIterator< Fragment_, Iterations_, AccessType_ > const &rhs_)
Create from non-constant FragmentIterator.
Definition: fragment.h:254
cutlass::FragmentIterator::kElementsPerAccess
static int const kElementsPerAccess
The number of elements per access.
Definition: fragment.h:183
cutlass::FragmentIterator::Strides
ShapeStrides< FragmentShape >::Shape Strides
The linear strides for iterations.
Definition: fragment.h:187
shape.h
Defines Shape implementing the Layout concept for representing a 4D hypercube of objects.
cutlass::FragmentIterator::AccessType
AccessType_ AccessType
The access type.
Definition: fragment.h:178
cutlass::FragmentIterator::valid
CUTLASS_DEVICE bool valid(int d, int h, int w, int c) const
Is the iterator valid?
Definition: fragment.h:219
cutlass::StorageType::Type
uint64_t Type
Definition: fragment.h:77
cutlass::is_pow2
Definition: cutlass_math.h:45
cutlass::FragmentConstIterator::FragmentConstIterator
CUTLASS_DEVICE FragmentConstIterator(OtherFragment_ &fragment, int offset=0)
Ctor.
Definition: fragment.h:249
cutlass::FragmentIterator::at
CUTLASS_DEVICE AccessType const & at(int d, int h, int w, int c=0) const
The accessor.
Definition: fragment.h:197
cutlass::FragmentConstIterator::pointer
Element const * pointer
The pointer.
Definition: fragment.h:273