load_store.h

Go to the documentation of this file.

/***************************************************************************************************
 * Copyright (c) 2017, NVIDIA CORPORATION.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification, are permitted
 * provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright notice, this list of
 *       conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright notice, this list of
 *       conditions and the following disclaimer in the documentation and/or other materials
 *       provided with the distribution.
 *     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
 *       to endorse or promote products derived from this software without specific prior written
 *       permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
 * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 **************************************************************************************************/
#pragma once

#include <cutlass/vector.h>

namespace cutlass {


struct MemorySpace {
  enum Kind {
    kGeneric,  // Data accessed through pointer dereferencing
    kShared,   // Data resides in shared memory
    kGlobal    // Data resides in global memory
  };
};


template <typename Scalar_,
          int Lanes_,
          MemorySpace::Kind Memory_,
          bool = (Lanes_ > 1),
          size_t = (sizeof(Scalar_) * Lanes_)>
struct Load {
  typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType;

  static CUTLASS_DEVICE void load(AccessType& dst, Scalar_ const* pointer, int offset) {
    dst = reinterpret_cast<AccessType const*>(&pointer[offset])[0];
  }
};


template <typename Scalar_, int Lanes_, MemorySpace::Kind Memory_>
struct Load<Scalar_, Lanes_, Memory_, true, 4> {
  typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType;

  static CUTLASS_DEVICE void load(AccessType& dst, Scalar_ const* pointer, int offset) {
    dst.registers[0] = reinterpret_cast<uint32_t const*>(&pointer[offset])[0];
  }
};


template <typename Scalar_, int Lanes_, MemorySpace::Kind Memory_>
struct Load<Scalar_, Lanes_, Memory_, true, 8> {
  typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType;

  static CUTLASS_DEVICE void load(AccessType& dst, Scalar_ const* pointer, int offset) {
    uint2 tmp = reinterpret_cast<uint2 const*>(&pointer[offset])[0];
    dst.registers[0] = tmp.x;
    dst.registers[1] = tmp.y;
  }
};


template <MemorySpace::Kind Memory_>
struct Load<double, 2, Memory_, true, 16> {
  typedef typename Vectorize<double, 2>::Type AccessType;

  static CUTLASS_DEVICE void load(AccessType& dst, double const* pointer, int offset) {
    double2 tmp = reinterpret_cast<double2 const*>(&pointer[offset])[0];
    dst[0] = tmp.x;
    dst[1] = tmp.y;
  }
};


template <typename Scalar_, int Lanes_, MemorySpace::Kind Memory_>
struct Load<Scalar_, Lanes_, Memory_, true, 16> {
  typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType;

  static CUTLASS_DEVICE void load(AccessType& dst, Scalar_ const* pointer, int offset) {
    uint4 tmp = reinterpret_cast<uint4 const*>(&pointer[offset])[0];
    dst.registers[0] = tmp.x;
    dst.registers[1] = tmp.y;
    dst.registers[2] = tmp.z;
    dst.registers[3] = tmp.w;
  }
};


template <typename Scalar_,
          int Lanes_,
          MemorySpace::Kind Memory_,
          bool = (Lanes_ > 1),
          size_t = (sizeof(Scalar_) * Lanes_)>
struct Store {
  typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType;

  static CUTLASS_DEVICE void store(AccessType const& src, Scalar_* pointer, int offset) {
    pointer[offset] = src;
  }
};


template <typename Scalar_, int Lanes_, MemorySpace::Kind Memory_>
struct Store<Scalar_, Lanes_, Memory_, true, 4> {
  typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType;

  static CUTLASS_DEVICE void store(AccessType const& src, Scalar_* pointer, int offset) {
    uint32_t* addr = reinterpret_cast<uint32_t*>(&pointer[offset]);
    addr[0] = src.registers[0];
  }
};


template <typename Scalar_, int Lanes_, MemorySpace::Kind Memory_>
struct Store<Scalar_, Lanes_, Memory_, true, 8> {
  typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType;

  static CUTLASS_DEVICE void store(AccessType const& src, Scalar_* pointer, int offset) {
    uint2* addr = reinterpret_cast<uint2*>(&pointer[offset]);
    addr[0] = make_uint2(src.registers[0], src.registers[1]);
  }
};


template <MemorySpace::Kind Memory_>
struct Store<double, 2, Memory_, true, 16> {
  typedef typename Vectorize<double, 2>::Type AccessType;

  static CUTLASS_DEVICE void store(AccessType const& src, double* pointer, int offset) {
    double2* addr = reinterpret_cast<double2*>(&pointer[offset]);
    addr[0] = make_double2(src[0], src[1]);
  }
};


template <typename Scalar_, int Lanes_, MemorySpace::Kind Memory_>
struct Store<Scalar_, Lanes_, Memory_, true, 16> {
  typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType;

  static CUTLASS_DEVICE void store(AccessType const& src, Scalar_* pointer, int offset) {
    uint4* addr = reinterpret_cast<uint4*>(&pointer[offset]);
    addr[0] = make_uint4(src.registers[0], src.registers[1], src.registers[2], src.registers[3]);
  }
};


}  // namespace cutlass