281 lines
9.7 KiB
Plaintext
281 lines
9.7 KiB
Plaintext
/***************************************************************************************************
|
|
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
|
* provided that the following conditions are met:
|
|
* * Redistributions of source code must retain the above copyright notice, this list of
|
|
* conditions and the following disclaimer.
|
|
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
|
* conditions and the following disclaimer in the documentation and/or other materials
|
|
* provided with the distribution.
|
|
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
|
* to endorse or promote products derived from this software without specific prior written
|
|
* permission.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
|
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
|
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
|
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
|
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
|
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
*
|
|
**************************************************************************************************/
|
|
/*! \file
|
|
\brief Unit tests for thread-level GEMM
|
|
*/
|
|
|
|
#include "../../common/cutlass_unit_test.h"
|
|
|
|
#include "cutlass/epilogue/thread/linear_combination_planar_complex.h"
|
|
|
|
/////////////////////////////////////////////////////////////////////////////////////////////////
|
|
|
|
namespace test {
|
|
namespace epilogue {
|
|
namespace thread {
|
|
|
|
using FunctorPlanarComplexF32F32 = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
|
|
float,
|
|
4,
|
|
float,
|
|
float>;
|
|
|
|
__global__ void epilogue_thread_functor_planar_complex_f32_f32(
|
|
float *output_ptr,
|
|
float const *accum_ptr,
|
|
float const *source_ptr,
|
|
typename FunctorPlanarComplexF32F32::Params params) {
|
|
|
|
FunctorPlanarComplexF32F32 linear_combination_op(params);
|
|
|
|
auto accum = *reinterpret_cast<cutlass::ArrayPlanarComplex<float , 4> const *>(accum_ptr);
|
|
auto source = *reinterpret_cast<cutlass::ArrayPlanarComplex<float, 4> const *>(source_ptr);
|
|
|
|
*reinterpret_cast<cutlass::ArrayPlanarComplex<float, 4>*>(output_ptr) = linear_combination_op(accum, source);
|
|
}
|
|
|
|
}
|
|
}
|
|
}
|
|
|
|
/////////////////////////////////////////////////////////////////////////////////////////////////
|
|
|
|
TEST(Epilogue_thread_linear_combination_planar_complex, f32) {
|
|
|
|
using Element = float;
|
|
using ElementOutput = float;
|
|
int const kCount = 4;
|
|
|
|
using Functor = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
|
|
ElementOutput,
|
|
kCount,
|
|
Element,
|
|
Element>;
|
|
|
|
cutlass::complex<Element> alpha(Element(2), Element(1));
|
|
cutlass::complex<Element> beta(Element(1), Element(-1));
|
|
|
|
typename Functor::Params params(alpha, beta);
|
|
|
|
Functor linear_combination_op(params);
|
|
|
|
cutlass::ArrayPlanarComplex<ElementOutput, kCount> source;
|
|
cutlass::ArrayPlanarComplex<Element, kCount> accum;
|
|
|
|
// Define arbitrary inputs
|
|
for (int i = 0; i < kCount; ++i) {
|
|
accum.real[i] = Element(i * 2);
|
|
accum.imag[i] = Element((i * 3 % 6) - 3);
|
|
source.real[i] = ElementOutput((i * 7 % 9) - 4);
|
|
source.imag[i] = ElementOutput(((i * 5 + 2) % 9) - 4);
|
|
}
|
|
|
|
cutlass::ArrayPlanarComplex<ElementOutput, kCount> destination = linear_combination_op(accum, source);
|
|
|
|
// Verify each result
|
|
for (int i = 0; i < kCount; ++i) {
|
|
|
|
cutlass::complex<Element> expected = alpha * cutlass::complex<Element>(accum.real[i], accum.imag[i]) +
|
|
beta * cutlass::complex<Element>(Element(source.real[i]), Element(source.imag[i]));
|
|
|
|
cutlass::complex<ElementOutput> got(destination.real[i], destination.imag[i]);
|
|
|
|
EXPECT_TRUE(ElementOutput(expected.real()) == got.real());
|
|
EXPECT_TRUE(ElementOutput(expected.imag()) == got.imag());
|
|
EXPECT_TRUE(expected.real() != Element(0) || expected.imag() != Element(0));
|
|
}
|
|
}
|
|
|
|
/////////////////////////////////////////////////////////////////////////////////////////////////
|
|
|
|
namespace test {
|
|
namespace epilogue {
|
|
namespace thread {
|
|
|
|
using FunctorPlanarComplexF16F32 = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
|
|
cutlass::half_t,
|
|
4,
|
|
float,
|
|
float>;
|
|
|
|
__global__ void epilogue_thread_functor_planar_complex_f16_f32(
|
|
cutlass::half_t *output_ptr,
|
|
float const *accum_ptr,
|
|
cutlass::half_t const *source_ptr,
|
|
typename FunctorPlanarComplexF16F32::Params params,
|
|
int N) {
|
|
|
|
FunctorPlanarComplexF16F32 linear_combination_op(params);
|
|
|
|
|
|
auto accum = *reinterpret_cast<cutlass::ArrayPlanarComplex<float , 4> const *>(accum_ptr);
|
|
auto source = *reinterpret_cast<cutlass::ArrayPlanarComplex<cutlass::half_t , 4> const *>(source_ptr);
|
|
|
|
#pragma unroll 1
|
|
for (int n = 0; n < N; ++n) {
|
|
source = linear_combination_op(accum, source);
|
|
}
|
|
|
|
*reinterpret_cast<cutlass::ArrayPlanarComplex<cutlass::half_t , 4>*>(output_ptr) = source;
|
|
}
|
|
|
|
}
|
|
}
|
|
}
|
|
|
|
/////////////////////////////////////////////////////////////////////////////////////////////////
|
|
|
|
TEST(Epilogue_thread_linear_combination_planar_complex, f16_f32) {
|
|
|
|
using Element = float;
|
|
using ElementOutput = cutlass::half_t;
|
|
int const kCount = 4;
|
|
|
|
using Functor = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
|
|
ElementOutput,
|
|
kCount,
|
|
Element,
|
|
Element>;
|
|
|
|
cutlass::complex<Element> alpha(Element(2), Element(1));
|
|
cutlass::complex<Element> beta(Element(1), Element(-1));
|
|
|
|
typename Functor::Params params(alpha, beta);
|
|
|
|
Functor linear_combination_op(params);
|
|
|
|
cutlass::ArrayPlanarComplex<ElementOutput, kCount> source;
|
|
cutlass::ArrayPlanarComplex<Element, kCount> accum;
|
|
|
|
// Define arbitrary inputs
|
|
for (int i = 0; i < kCount; ++i) {
|
|
accum.real[i] = Element(i * 2);
|
|
accum.imag[i] = Element((i * 3 % 6) - 3);
|
|
source.real[i] = ElementOutput((i * 7 % 9) - 4);
|
|
source.imag[i] = ElementOutput(((i * 5 + 2) % 9) - 4);
|
|
}
|
|
|
|
cutlass::ArrayPlanarComplex<ElementOutput, kCount> destination = linear_combination_op(accum, source);
|
|
|
|
// Verify each result
|
|
for (int i = 0; i < kCount; ++i) {
|
|
|
|
cutlass::complex<Element> expected = alpha * cutlass::complex<Element>(accum.real[i], accum.imag[i]) +
|
|
beta * cutlass::complex<Element>(Element(source.real[i]), Element(source.imag[i]));
|
|
|
|
cutlass::complex<ElementOutput> got(destination.real[i], destination.imag[i]);
|
|
|
|
EXPECT_TRUE(ElementOutput(expected.real()) == got.real());
|
|
EXPECT_TRUE(ElementOutput(expected.imag()) == got.imag());
|
|
EXPECT_TRUE(expected.real() != Element(0) || expected.imag() != Element(0));
|
|
}
|
|
}
|
|
|
|
/////////////////////////////////////////////////////////////////////////////////////////////////
|
|
|
|
namespace test {
|
|
namespace epilogue {
|
|
namespace thread {
|
|
|
|
using FunctorPlanarComplexF16F16 = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
|
|
cutlass::half_t,
|
|
4,
|
|
cutlass::half_t,
|
|
cutlass::half_t>;
|
|
|
|
__global__ void epilogue_thread_functor_planar_complex_f16_f16(
|
|
cutlass::half_t *output_ptr,
|
|
cutlass::half_t const *accum_ptr,
|
|
cutlass::half_t const *source_ptr,
|
|
typename FunctorPlanarComplexF16F16::Params params,
|
|
int N) {
|
|
|
|
FunctorPlanarComplexF16F16 linear_combination_op(params);
|
|
|
|
auto accum = *reinterpret_cast<cutlass::ArrayPlanarComplex<cutlass::half_t , 4> const *>(accum_ptr);
|
|
auto source = *reinterpret_cast<cutlass::ArrayPlanarComplex<cutlass::half_t , 4> const *>(source_ptr);
|
|
|
|
#pragma unroll 1
|
|
for (int n = 0; n < N; ++n) {
|
|
source = linear_combination_op(accum, source);
|
|
}
|
|
|
|
*reinterpret_cast<cutlass::ArrayPlanarComplex<cutlass::half_t , 4>*>(output_ptr) = source;
|
|
}
|
|
|
|
}
|
|
}
|
|
}
|
|
|
|
/////////////////////////////////////////////////////////////////////////////////////////////////
|
|
|
|
TEST(Epilogue_thread_linear_combination_planar_complex, f16_f16) {
|
|
|
|
using Element = cutlass::half_t;
|
|
using ElementOutput = cutlass::half_t;
|
|
int const kCount = 8;
|
|
|
|
using Functor = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
|
|
ElementOutput,
|
|
kCount,
|
|
Element,
|
|
Element>;
|
|
|
|
cutlass::complex<Element> alpha(Element(2), Element(1));
|
|
cutlass::complex<Element> beta(Element(1), Element(-1));
|
|
|
|
typename Functor::Params params(alpha, beta);
|
|
|
|
Functor linear_combination_op(params);
|
|
|
|
cutlass::ArrayPlanarComplex<ElementOutput, kCount> source;
|
|
cutlass::ArrayPlanarComplex<Element, kCount> accum;
|
|
|
|
// Define arbitrary inputs
|
|
for (int i = 0; i < kCount; ++i) {
|
|
accum.real[i] = Element(i * 2);
|
|
accum.imag[i] = Element((i * 3 % 6) - 3);
|
|
source.real[i] = ElementOutput((i * 7 % 9) - 4);
|
|
source.imag[i] = ElementOutput(((i * 5 + 2) % 9) - 4);
|
|
}
|
|
|
|
cutlass::ArrayPlanarComplex<ElementOutput, kCount> destination = linear_combination_op(accum, source);
|
|
|
|
// Verify each result
|
|
for (int i = 0; i < kCount; ++i) {
|
|
|
|
cutlass::complex<Element> expected = alpha * cutlass::complex<Element>(accum.real[i], accum.imag[i]) +
|
|
beta * cutlass::complex<Element>(Element(source.real[i]), Element(source.imag[i]));
|
|
|
|
cutlass::complex<ElementOutput> got(destination.real[i], destination.imag[i]);
|
|
|
|
EXPECT_TRUE(ElementOutput(expected.real()) == got.real());
|
|
EXPECT_TRUE(ElementOutput(expected.imag()) == got.imag());
|
|
EXPECT_TRUE(expected.real() != Element(0) || expected.imag() != Element(0));
|
|
}
|
|
}
|
|
|
|
/////////////////////////////////////////////////////////////////////////////////////////////////
|