Skip to content

Commit

Permalink
ENH: Vectorize FP16 math functions on Intel SkylakeX
Browse files Browse the repository at this point in the history
  • Loading branch information
@r-devulap
r-devulap committed Sep 26, 2022
1 parent 0e87c99 commit 8dd6761
Show file tree
Hide file tree
Showing 4 changed files with 87 additions and 38 deletions.
57 changes: 20 additions & 37 deletions numpy/core/code_generators/generate_umath.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -645,108 +645,97 @@ def english_upper(s):
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.arccos'),
None,
TD('e', f='acos', astype={'e': 'f'}),
TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='acos', astype={'e': 'f'}),
TD(P, f='arccos'),
),
'arccosh':
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.arccosh'),
None,
TD('e', f='acosh', astype={'e': 'f'}),
TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='acosh', astype={'e': 'f'}),
TD(P, f='arccosh'),
),
'arcsin':
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.arcsin'),
None,
TD('e', f='asin', astype={'e': 'f'}),
TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='asin', astype={'e': 'f'}),
TD(P, f='arcsin'),
),
'arcsinh':
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.arcsinh'),
None,
TD('e', f='asinh', astype={'e': 'f'}),
TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='asinh', astype={'e': 'f'}),
TD(P, f='arcsinh'),
),
'arctan':
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.arctan'),
None,
TD('e', f='atan', astype={'e': 'f'}),
TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='atan', astype={'e': 'f'}),
TD(P, f='arctan'),
),
'arctanh':
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.arctanh'),
None,
TD('e', f='atanh', astype={'e': 'f'}),
TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='atanh', astype={'e': 'f'}),
TD(P, f='arctanh'),
),
'cos':
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.cos'),
None,
TD('e', f='cos', astype={'e': 'f'}),
TD('f', dispatch=[('loops_trigonometric', 'f')]),
TD('d', dispatch=[('loops_umath_fp', 'd')]),
TD('ed', dispatch=[('loops_umath_fp', 'ed')]),
TD('fdg' + cmplx, f='cos'),
TD(P, f='cos'),
),
'sin':
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.sin'),
None,
TD('e', f='sin', astype={'e': 'f'}),
TD('f', dispatch=[('loops_trigonometric', 'f')]),
TD('d', dispatch=[('loops_umath_fp', 'd')]),
TD('ed', dispatch=[('loops_umath_fp', 'ed')]),
TD('fdg' + cmplx, f='sin'),
TD(P, f='sin'),
),
'tan':
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.tan'),
None,
TD('e', f='tan', astype={'e': 'f'}),
TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='tan', astype={'e': 'f'}),
TD(P, f='tan'),
),
'cosh':
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.cosh'),
None,
TD('e', f='cosh', astype={'e': 'f'}),
TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='cosh', astype={'e': 'f'}),
TD(P, f='cosh'),
),
'sinh':
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.sinh'),
None,
TD('e', f='sinh', astype={'e': 'f'}),
TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='sinh', astype={'e': 'f'}),
TD(P, f='sinh'),
),
'tanh':
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.tanh'),
None,
TD('e', f='tanh', astype={'e': 'f'}),
TD('e', dispatch=[('loops_umath_fp', 'e')]),
TD('fd', dispatch=[('loops_hyperbolic', 'fd')]),
TD(inexact, f='tanh', astype={'e': 'f'}),
TD(P, f='tanh'),
Expand All @@ -755,7 +744,7 @@ def english_upper(s):
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.exp'),
None,
TD('e', f='exp', astype={'e': 'f'}),
TD('e', dispatch=[('loops_umath_fp', 'e')]),
TD('fd', dispatch=[('loops_exponent_log', 'fd')]),
TD('fdg' + cmplx, f='exp'),
TD(P, f='exp'),
Expand All @@ -764,25 +753,23 @@ def english_upper(s):
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.exp2'),
None,
TD('e', f='exp2', astype={'e': 'f'}),
TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='exp2', astype={'e': 'f'}),
TD(P, f='exp2'),
),
'expm1':
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.expm1'),
None,
TD('e', f='expm1', astype={'e': 'f'}),
TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='expm1', astype={'e': 'f'}),
TD(P, f='expm1'),
),
'log':
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.log'),
None,
TD('e', f='log', astype={'e': 'f'}),
TD('e', dispatch=[('loops_umath_fp', 'e')]),
TD('fd', dispatch=[('loops_exponent_log', 'fd')]),
TD('fdg' + cmplx, f='log'),
TD(P, f='log'),
Expand All @@ -791,26 +778,23 @@ def english_upper(s):
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.log2'),
None,
TD('e', f='log2', astype={'e': 'f'}),
TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='log2', astype={'e': 'f'}),
TD(P, f='log2'),
),
'log10':
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.log10'),
None,
TD('e', f='log10', astype={'e': 'f'}),
TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='log10', astype={'e': 'f'}),
TD(P, f='log10'),
),
'log1p':
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.log1p'),
None,
TD('e', f='log1p', astype={'e': 'f'}),
TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='log1p', astype={'e': 'f'}),
TD(P, f='log1p'),
),
Expand All @@ -827,8 +811,7 @@ def english_upper(s):
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.cbrt'),
None,
TD('e', f='cbrt', astype={'e': 'f'}),
TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(flts, f='cbrt', astype={'e': 'f'}),
TD(P, f='cbrt'),
),
Expand Down
1 change: 0 additions & 1 deletion numpy/core/setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1077,7 +1077,6 @@ def generate_umath_doc_header(ext, build_dir):
# actually the other way around, better performance and
# after all maintainable code.
svml_filter = (
'svml_z0_tanh_d_la.s', 'svml_z0_tanh_s_la.s'
)
if can_link_svml() and check_svml_submodule(svml_path):
svml_objs = glob.glob(svml_path + '/**/*.s', recursive=True)
Expand Down
9 changes: 9 additions & 0 deletions numpy/core/src/umath/loops.h.src
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,15 @@ NPY_CPU_DISPATCH_DECLARE(NPY_NO_EXPORT void @TYPE@_@func@,
/**end repeat1**/
/**end repeat**/

/**begin repeat
* #func = sin, cos, tan, exp, exp2, log, log2, log10, expm1, log1p, cbrt, arcsin, arccos, arctan, sinh, cosh, tanh, arcsinh, arccosh, arctanh#
*/

NPY_CPU_DISPATCH_DECLARE(NPY_NO_EXPORT void HALF_@func@,
(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)))

/**end repeat**/

/**begin repeat
* #func = sin, cos#
*/
Expand Down
58 changes: 58 additions & 0 deletions numpy/core/src/umath/loops_umath_fp.dispatch.c.src
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,64 @@ simd_@func@_@sfx@(const npyv_lanetype_@sfx@ *src1, npy_intp ssrc1,

#endif

#if NPY_SIMD && defined(NPY_HAVE_AVX512_SKX) && defined(NPY_CAN_LINK_SVML)
/**begin repeat
* #func = sin, cos, tan, exp, exp2, expm1, log, log2, log10, log1p, cbrt, asin, acos, atan, sinh, cosh, tanh, asinh, acosh, atanh#
* #default_val = 0, 0, 0, 0, 0, 0x3c00, 0x3c00, 0x3c00, 0x3c00, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x3c00, 0#
*/
static void
avx512_@func@_f16(const npy_half *src, npy_half *dst, npy_intp len)
{
const int num_lanes = npyv_nlanes_f32;
npyvh_f16 x, out;
npyv_f32 x_ps, out_ps;
for (; len > 0; len -= num_lanes, src += num_lanes, dst += num_lanes) {
if (len >= num_lanes) {
x = npyvh_load_f16(src);
x_ps = npyv_cvt_f16_f32(x);
out_ps = __svml_@func@f16(x_ps);
out = npyv_cvt_f32_f16(out_ps, 0);
npyvh_store_f16(dst, out);
}
else {
x = npyvh_load_till_f16(src, len, @default_val@);
x_ps = npyv_cvt_f16_f32(x);
out_ps = __svml_@func@f16(x_ps);
out = npyv_cvt_f32_f16(out_ps, 0);
npyvh_store_till_f16(dst, len, out);
}
}
npyv_cleanup();
}
/**end repeat**/
#endif

/**begin repeat
* #func = sin, cos, tan, exp, exp2, expm1, log, log2, log10, log1p, cbrt, arcsin, arccos, arctan, sinh, cosh, tanh, arcsinh, arccosh, arctanh#
* #intrin = sin, cos, tan, exp, exp2, expm1, log, log2, log10, log1p, cbrt, asin, acos, atan, sinh, cosh, tanh, asinh, acosh, atanh#
*/
NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(HALF_@func@)
(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(data))
{
#if NPY_SIMD && defined(NPY_HAVE_AVX512_SKX) && defined(NPY_CAN_LINK_SVML)
const npy_half *src = (npy_half*)args[0];
npy_half *dst = (npy_half*)args[1];
const int lsize = sizeof(src[0]);
const npy_intp ssrc = steps[0] / lsize;
const npy_intp sdst = steps[1] / lsize;
const npy_intp len = dimensions[0];
if ((ssrc == 1) && (sdst == 1)) {
avx512_@intrin@_f16(src, dst, len);
return;
}
#endif
UNARY_LOOP {
const npy_float in1 = npy_half_to_float(*(npy_half *)ip1);
*((npy_half *)op1) = npy_float_to_half(npy_@intrin@f(in1));
}
}
/**end repeat**/

/**begin repeat
* #TYPE = DOUBLE, FLOAT#
* #type = npy_double, npy_float#
Expand Down

0 comments on commit 8dd6761

Please sign in to comment.