load_store.h
Go to the documentation of this file.
/***************************************************************************************************
* Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#pragma once
#include <cutlass/vector.h>
namespace cutlass {
struct MemorySpace {
enum Kind {
kGeneric, // Data accessed through pointer dereferencing
kShared, // Data resides in shared memory
kGlobal // Data resides in global memory
};
};
template <typename Scalar_,
int Lanes_,
MemorySpace::Kind Memory_,
bool = (Lanes_ > 1),
size_t = (sizeof(Scalar_) * Lanes_)>
struct Load {
typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType;
static CUTLASS_DEVICE void load(AccessType& dst, Scalar_ const* pointer, int offset) {
dst = reinterpret_cast< AccessType const*>(&pointer[offset])[0];
}
};
template <typename Scalar_, int Lanes_, MemorySpace::Kind Memory_>
struct Load<Scalar_, Lanes_, Memory_, true, 4> {
typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType;
static CUTLASS_DEVICE void load(AccessType& dst, Scalar_ const* pointer, int offset) {
dst.registers[0] = reinterpret_cast<uint32_t const*>(&pointer[offset])[0];
}
};
template <typename Scalar_, int Lanes_, MemorySpace::Kind Memory_>
struct Load<Scalar_, Lanes_, Memory_, true, 8> {
typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType;
static CUTLASS_DEVICE void load(AccessType& dst, Scalar_ const* pointer, int offset) {
uint2 tmp = reinterpret_cast<uint2 const*>(&pointer[offset])[0];
dst.registers[0] = tmp.x;
dst.registers[1] = tmp.y;
}
};
template <MemorySpace::Kind Memory_>
struct Load<double, 2, Memory_, true, 16> {
typedef typename Vectorize<double, 2>::Type AccessType;
static CUTLASS_DEVICE void load(AccessType& dst, double const* pointer, int offset) {
double2 tmp = reinterpret_cast<double2 const*>(&pointer[offset])[0];
dst[0] = tmp.x;
dst[1] = tmp.y;
}
};
template <typename Scalar_, int Lanes_, MemorySpace::Kind Memory_>
struct Load<Scalar_, Lanes_, Memory_, true, 16> {
typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType;
static CUTLASS_DEVICE void load(AccessType& dst, Scalar_ const* pointer, int offset) {
uint4 tmp = reinterpret_cast<uint4 const*>(&pointer[offset])[0];
dst.registers[0] = tmp.x;
dst.registers[1] = tmp.y;
dst.registers[2] = tmp.z;
dst.registers[3] = tmp.w;
}
};
template <typename Scalar_,
int Lanes_,
MemorySpace::Kind Memory_,
bool = (Lanes_ > 1),
size_t = (sizeof(Scalar_) * Lanes_)>
struct Store {
typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType;
static CUTLASS_DEVICE void store(AccessType const& src, Scalar_* pointer, int offset) {
pointer[offset] = src;
}
};
template <typename Scalar_, int Lanes_, MemorySpace::Kind Memory_>
struct Store<Scalar_, Lanes_, Memory_, true, 4> {
typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType;
static CUTLASS_DEVICE void store(AccessType const& src, Scalar_* pointer, int offset) {
uint32_t* addr = reinterpret_cast<uint32_t*>(&pointer[offset]);
addr[0] = src.registers[0];
}
};
template <typename Scalar_, int Lanes_, MemorySpace::Kind Memory_>
struct Store<Scalar_, Lanes_, Memory_, true, 8> {
typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType;
static CUTLASS_DEVICE void store(AccessType const& src, Scalar_* pointer, int offset) {
uint2* addr = reinterpret_cast<uint2*>(&pointer[offset]);
addr[0] = make_uint2(src.registers[0], src.registers[1]);
}
};
template <MemorySpace::Kind Memory_>
struct Store<double, 2, Memory_, true, 16> {
typedef typename Vectorize<double, 2>::Type AccessType;
static CUTLASS_DEVICE void store(AccessType const& src, double* pointer, int offset) {
double2* addr = reinterpret_cast<double2*>(&pointer[offset]);
addr[0] = make_double2(src[0], src[1]);
}
};
template <typename Scalar_, int Lanes_, MemorySpace::Kind Memory_>
struct Store<Scalar_, Lanes_, Memory_, true, 16> {
typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType;
static CUTLASS_DEVICE void store(AccessType const& src, Scalar_* pointer, int offset) {
uint4* addr = reinterpret_cast<uint4*>(&pointer[offset]);
addr[0] = make_uint4(src.registers[0], src.registers[1], src.registers[2], src.registers[3]);
}
};
} // namespace cutlass
cutlass::Store< Scalar_, Lanes_, Memory_, true, 16 >::AccessType
Vectorize< Scalar_, Lanes_ >::Type AccessType
The output type.
Definition: load_store.h:188
cutlass::MemorySpace::kShared
Definition: load_store.h:42
cutlass
Definition: convert.h:33
cutlass::Store< Scalar_, Lanes_, Memory_, true, 8 >::store
static CUTLASS_DEVICE void store(AccessType const &src, Scalar_ *pointer, int offset)
The store function.
Definition: load_store.h:163
cutlass::MemorySpace
Enum to specify which memory space data resides in.
Definition: load_store.h:39
cutlass::MemorySpace::kGlobal
Definition: load_store.h:43
cutlass::Load::load
static CUTLASS_DEVICE void load(AccessType &dst, Scalar_ const *pointer, int offset)
The load function.
Definition: load_store.h:59
cutlass::Load< Scalar_, Lanes_, Memory_, true, 16 >::AccessType
Vectorize< Scalar_, Lanes_ >::Type AccessType
The output type.
Definition: load_store.h:112
cutlass::Store< Scalar_, Lanes_, Memory_, true, 4 >::AccessType
Vectorize< Scalar_, Lanes_ >::Type AccessType
The output type.
Definition: load_store.h:146
cutlass::MemorySpace::Kind
Kind
Definition: load_store.h:40
cutlass::Store
Definition: load_store.h:131
cutlass::Store::store
static CUTLASS_DEVICE void store(AccessType const &src, Scalar_ *pointer, int offset)
The store function.
Definition: load_store.h:136
cutlass::Vector::registers
uint32_t registers[kRegisters]
The data in registers.
Definition: vector.h:80
cutlass::Store< double, 2, Memory_, true, 16 >::AccessType
Vectorize< double, 2 >::Type AccessType
The output type.
Definition: load_store.h:174
cutlass::MemorySpace::kGeneric
Definition: load_store.h:41
cutlass::Load< Scalar_, Lanes_, Memory_, true, 4 >::load
static CUTLASS_DEVICE void load(AccessType &dst, Scalar_ const *pointer, int offset)
The store function.
Definition: load_store.h:72
cutlass::Store::AccessType
Vectorize< Scalar_, Lanes_ >::Type AccessType
The output type.
Definition: load_store.h:133
cutlass::Vector
Definition: vector.h:61
cutlass::Load< Scalar_, Lanes_, Memory_, true, 8 >::load
static CUTLASS_DEVICE void load(AccessType &dst, Scalar_ const *pointer, int offset)
The store function.
Definition: load_store.h:85
cutlass::Load
Definition: load_store.h:54
cutlass::Load< Scalar_, Lanes_, Memory_, true, 8 >::AccessType
Vectorize< Scalar_, Lanes_ >::Type AccessType
The output type.
Definition: load_store.h:82
vector.h
Defines a 1D vector of elements held in the registers of each thread.
cutlass::Store< Scalar_, Lanes_, Memory_, true, 8 >::AccessType
Vectorize< Scalar_, Lanes_ >::Type AccessType
The output type.
Definition: load_store.h:160
cutlass::Load< Scalar_, Lanes_, Memory_, true, 16 >::load
static CUTLASS_DEVICE void load(AccessType &dst, Scalar_ const *pointer, int offset)
The store function.
Definition: load_store.h:115
cutlass::Load< Scalar_, Lanes_, Memory_, true, 4 >::AccessType
Vectorize< Scalar_, Lanes_ >::Type AccessType
The output type.
Definition: load_store.h:69
cutlass::Load< double, 2, Memory_, true, 16 >::load
static CUTLASS_DEVICE void load(AccessType &dst, double const *pointer, int offset)
The store function.
Definition: load_store.h:100
cutlass::Load< double, 2, Memory_, true, 16 >::AccessType
Vectorize< double, 2 >::Type AccessType
The output type.
Definition: load_store.h:97
cutlass::Load::AccessType
Vectorize< Scalar_, Lanes_ >::Type AccessType
The output type.
Definition: load_store.h:56
cutlass::Store< Scalar_, Lanes_, Memory_, true, 16 >::store
static CUTLASS_DEVICE void store(AccessType const &src, Scalar_ *pointer, int offset)
The store function.
Definition: load_store.h:191
cutlass::Store< Scalar_, Lanes_, Memory_, true, 4 >::store
static CUTLASS_DEVICE void store(AccessType const &src, Scalar_ *pointer, int offset)
The store function.
Definition: load_store.h:149
cutlass::Store< double, 2, Memory_, true, 16 >::store
static CUTLASS_DEVICE void store(AccessType const &src, double *pointer, int offset)
The store function.
Definition: load_store.h:177