squall/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

replace division with multiplication in GELU (#942)

Browse Source
This commit is contained in:
wll 2023-05-12 22:57:18 +08:00 committed by GitHub
parent fcfbd23e26
commit 19c4a4815e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/cutlass/epilogue/thread/activation.h
View File

@ -515,7 +515,7 @@ struct GELU {
CUTLASS_HOST_DEVICE CUTLASS_HOST_DEVICE
T operator()(T const &scalar) const { T operator()(T const &scalar) const {
return T(cutlass::constants::half<T>() * scalar * return T(cutlass::constants::half<T>() * scalar *
(cutlass::constants::one<T>() + (T)erff((float)(scalar / cutlass::constants::root_two<T>())))); (cutlass::constants::one<T>() + (T)erff((float)(scalar * cutlass::constants::half_root_two<T>()))));
} }
using Params = LinearCombinationGenericParams<T>; using Params = LinearCombinationGenericParams<T>;
@ -531,7 +531,7 @@ struct GELU<float> {
CUTLASS_HOST_DEVICE CUTLASS_HOST_DEVICE
float operator()(float const &scalar) const { float operator()(float const &scalar) const {
return cutlass::constants::half<float>() * scalar * return cutlass::constants::half<float>() * scalar *
(cutlass::constants::one<float>() + erff( scalar / cutlass::constants::root_two<float>() )); (cutlass::constants::one<float>() + erff( scalar * cutlass::constants::half_root_two<float>() ));
} }
using Params = LinearCombinationGenericParams<float>; using Params = LinearCombinationGenericParams<float>;
@ -547,7 +547,7 @@ struct GELU<double> {
CUTLASS_HOST_DEVICE CUTLASS_HOST_DEVICE
double operator()(double const &scalar) const { double operator()(double const &scalar) const {
return cutlass::constants::half<double>() * scalar * return cutlass::constants::half<double>() * scalar *
(cutlass::constants::one<double>() + erf( scalar / cutlass::constants::root_two<double>() )); (cutlass::constants::one<double>() + erf( scalar * cutlass::constants::half_root_two<double>() ));
} }
using Params = LinearCombinationGenericParams<double>; using Params = LinearCombinationGenericParams<double>;